Return std::size_t from Body::writer::put (API Change)

`put` returns the number of bytes actually transferred
from the input buffers.

Actions Required:

* Return the number of bytes actually transferred from the
  input buffers in user defined `Body::writer::put` functions.

CHANGELOG.md

@@ -15,6 +15,15 @@ WebSockets:
 * Fix websocket write op
 * Add cmake options for examples and tests
 
+API Changes:
+
+* Return `std::size_t` from `Body::writer::put`
+
+Actions Required:
+
+* Return the number of bytes actually transferred from the
+  input buffers in user defined `Body::writer::put` functions.
+
 --------------------------------------------------------------------------------
 
 Version 70:

CMakeLists.txt

@@ -174,6 +174,10 @@ file(GLOB_RECURSE COMMON_INCLUDES
     ${PROJECT_SOURCE_DIR}/example/common/*.hpp
   )
 
+file(GLOB_RECURSE EXAMPLE_INCLUDES
+    ${PROJECT_SOURCE_DIR}/example/*.hpp
+  )
+
 file(GLOB_RECURSE EXTRAS_INCLUDES
     ${PROJECT_SOURCE_DIR}/extras/beast/*.hpp
     ${PROJECT_SOURCE_DIR}/extras/beast/*.ipp

doc/5_05_parser_streams.qbk

@@ -120,4 +120,19 @@ the response:
 
 [http_snippet_13]
 
+
+
+[section Incremental Read]
+
+This function uses
+[link beast.ref.beast__http__buffer_body `buffer_body`]
+and parser stream operations to read a message body progressively
+using a small, fixed-size buffer:
+
+[example_incremental_read]
+
+[endsect]
+
+
+
 [endsect]

doc/concept/BodyWriter.qbk

@@ -57,10 +57,13 @@ In this table:
     ]
 ][
     [`a.put(b,ec)`]
-    []
+    [`std::size_t`]
     [
-        This function is called to append the buffers specified by `b`
+        This function is called to append some or all of the buffers
-        into the body representation.
+        specified by `b` into the body representation. The number of
+        bytes inserted from `b` is returned. If the number of bytes
+        inserted is less than the total input, the remainder of the
+        input will be presented in the next call to `put`.
         The function will ensure that `!ec` is `true` if there was
         no error or set to the appropriate error code if there was one. 
     ]

example/common/file_body.hpp

@@ -255,7 +255,7 @@ public:
     // buffer sequences corresponding to the incoming body.
     //
     template<class ConstBufferSequence>
-    void
+    std::size_t
     put(ConstBufferSequence const& buffers, beast::error_code& ec);
 
     // This function is called when writing is complete.
@@ -307,16 +307,20 @@ writer(beast::http::message<isRequest, file_body, Fields>& m,
 // This will get called one or more times with body buffers
 //
 template<class ConstBufferSequence>
-void
+std::size_t
 file_body::writer::
 put(ConstBufferSequence const& buffers, beast::error_code& ec)
 {
+    // This function must return the total number of
+    // bytes transferred from the input buffers.
+    std::size_t bytes_transferred = 0;
+
     // Loop over all the buffers in the sequence,
     // and write each one to the file.
     for(boost::asio::const_buffer buffer : buffers)
     {
         // Write this buffer to the file
-        fwrite(
+        bytes_transferred += fwrite(
             boost::asio::buffer_cast<void const*>(buffer), 1,
             boost::asio::buffer_size(buffer),
             file_);
@@ -327,12 +331,14 @@ put(ConstBufferSequence const& buffers, beast::error_code& ec)
             // Convert the old-school `errno` into
             // an error code using the generic category.
             ec = beast::error_code{errno, beast::generic_category()};
-            return;
+            return bytes_transferred;
         }
     }
 
     // Indicate success
     ec = {};
+
+    return bytes_transferred;
 }
 
 // Called after writing is done when there's no error.

example/common/mutable_body.hpp

@@ -120,7 +120,7 @@ struct mutable_body
         }
 
         template<class ConstBufferSequence>
-        void
+        std::size_t
         put(ConstBufferSequence const& buffers,
                 beast::error_code& ec)
         {
@@ -135,10 +135,10 @@ struct mutable_body
             catch(std::length_error const&)
             {
                 ec = beast::http::error::buffer_overflow;
-                return;
+                return 0;
             }
             ec.assign(0, ec.category());
-            buffer_copy(boost::asio::buffer(
+            return buffer_copy(boost::asio::buffer(
                 &body_[0] + len, n), buffers);
         }
 

example/doc/http_examples.hpp

@@ -6,6 +6,7 @@
 //
 
 #include <beast.hpp>
+#include <iostream>
 
 /*  This file contains the functions and classes found in the documentation
 
@@ -917,12 +918,15 @@ class custom_parser
         content_length,         // Content length if known, else `boost::none`
         error_code& ec);        // The error returned to the caller, if any
 
-    /// Called for each piece of the body, if a body exists.
+    /** Called for each piece of the body, if a body exists.
-    //
+      
-    //  If present, the chunked Transfer-Encoding will be removed
+        If present, the chunked Transfer-Encoding will be removed
-    //  before this callback is invoked.
+        before this callback is invoked. The function returns
-    //
+        the number of bytes consumed from the input buffer.
-    void
+        Any input octets not consumed will be will be presented
+        on subsequent calls.
+    */
+    std::size_t
     on_data(
         string_view s,          // A portion of the body
         error_code& ec);        // The error returned to the caller, if any
@@ -990,11 +994,12 @@ on_body(boost::optional<std::uint64_t> const& content_length,
 }
 
 template<bool isRequest>
-void custom_parser<isRequest>::
+std::size_t custom_parser<isRequest>::
 on_data(string_view s, error_code& ec)
 {
     boost::ignore_unused(s);
     ec = {};
+    return s.size();
 }
 
 template<bool isRequest>
@@ -1013,5 +1018,47 @@ on_complete(error_code& ec)
     ec = {};
 }
 
+//------------------------------------------------------------------------------
+//
+// Example: Incremental Read
+//
+//------------------------------------------------------------------------------
+
+//[example_incremental_read
+
+/*  This function reads a message using a fixed size buffer to hold
+    portions of the body, and prints the body contents to a `std::ostream`.
+*/
+template<
+    bool isRequest,
+    class SyncReadStream,
+    class DynamicBuffer>
+void
+read_and_print_body(
+    std::ostream& os,
+    SyncReadStream& stream,
+    DynamicBuffer& buffer,
+    error_code& ec)
+{
+    parser<isRequest, buffer_body> p;
+    read_header(stream, buffer, p, ec);
+    if(ec)
+        return;
+    while(! p.is_done())
+    {
+        char buf[512];
+        p.get().body.data = buf;
+        p.get().body.size = sizeof(buf);
+        read(stream, buffer, p, ec);
+        if(ec == error::need_buffer)
+            ec.assign(0, ec.category());
+        if(ec)
+            return;
+        os.write(buf, sizeof(buf) - p.get().body.size);
+    }
+}
+
+//]
+
 } // http
 } // beast

include/beast/http/buffer_body.hpp

@@ -161,25 +161,28 @@ struct buffer_body
         }
 
         template<class ConstBufferSequence>
-        void
+        std::size_t
         put(ConstBufferSequence const& buffers,
             error_code& ec)
         {
             using boost::asio::buffer_size;
             using boost::asio::buffer_copy;
-            auto const n = buffer_size(buffers);
+            if(! body_.data)
-            if(! body_.data || n > body_.size)
             {
                 ec = error::need_buffer;
-                return;
+                return 0;
             }
-            ec.assign(0, ec.category());
             auto const bytes_transferred =
                 buffer_copy(boost::asio::buffer(
                     body_.data, body_.size), buffers);
             body_.data = reinterpret_cast<char*>(
                 body_.data) + bytes_transferred;
             body_.size -= bytes_transferred;
+            if(bytes_transferred == buffer_size(buffers))
+                ec.assign(0, ec.category());
+            else
+                ec = error::need_buffer;
+            return bytes_transferred;
         }
 
         void

include/beast/http/dynamic_body.hpp

@@ -13,6 +13,7 @@
 #include <beast/http/error.hpp>
 #include <beast/http/message.hpp>
 #include <boost/optional.hpp>
+#include <algorithm>
 #include <utility>
 
 namespace beast {
@@ -86,26 +87,35 @@ struct basic_dynamic_body
         }
 
         template<class ConstBufferSequence>
-        void
+        std::size_t
         put(ConstBufferSequence const& buffers,
             error_code& ec)
         {
             using boost::asio::buffer_copy;
             using boost::asio::buffer_size;
+            auto const n = buffer_size(buffers);
+            if(body_.size() > body_.max_size() - n)
+            {
+                ec = error::buffer_overflow;
+                return 0;
+            }
             boost::optional<typename
                 DynamicBuffer::mutable_buffers_type> b;
             try
             {
-                b.emplace(body_.prepare(
+                b.emplace(body_.prepare((std::min)(n,
-                    buffer_size(buffers)));
+                    body_.max_size() - body_.size())));
             }
             catch(std::length_error const&)
             {
                 ec = error::buffer_overflow;
-                return;
+                return 0;
             }
             ec.assign(0, ec.category());
-            body_.commit(buffer_copy(*b, buffers));
+            auto const bytes_transferred =
+                buffer_copy(*b, buffers);
+            body_.commit(bytes_transferred);
+            return bytes_transferred;
         }
 
         void

include/beast/http/empty_body.hpp

@@ -85,11 +85,12 @@ struct empty_body
         }
 
         template<class ConstBufferSequence>
-        void
+        std::size_t
         put(ConstBufferSequence const&,
             error_code& ec)
         {
             ec = error::unexpected_body;
+            return 0;
         }
 
         void

include/beast/http/impl/basic_parser.ipp

@@ -523,12 +523,12 @@ basic_parser<isRequest, Derived>::
 parse_body(char const*& p,
     std::size_t n, error_code& ec)
 {
-    n = beast::detail::clamp(len_, n);
+    n = impl().on_data(string_view{p,
-    impl().on_data(string_view{p, n}, ec);
+        beast::detail::clamp(len_, n)}, ec);
-    if(ec)
-        return;
     p += n;
     len_ -= n;
+    if(ec)
+        return;
     if(len_ > 0)
         return;
     impl().on_complete(ec);
@@ -550,10 +550,10 @@ parse_body_to_eof(char const*& p,
         return;
     }
     body_limit_ = body_limit_ - n;
-    impl().on_data(string_view{p, n}, ec);
+    n = impl().on_data(string_view{p, n}, ec);
+    p += n;
     if(ec)
         return;
-    p += n;
 }
 
 template<bool isRequest, class Derived>
@@ -701,12 +701,12 @@ basic_parser<isRequest, Derived>::
 parse_chunk_body(char const*& p,
     std::size_t n, error_code& ec)
 {
-    n = beast::detail::clamp(len_, n);
+    n = impl().on_data(string_view{p,
-    impl().on_data(string_view{p, n}, ec);
+        beast::detail::clamp(len_, n)}, ec);
-    if(ec)
-        return;
     p += n;
     len_ -= n;
+    if(ec)
+        return;
     if(len_ > 0)
         return;
     state_ = state::chunk_header;

include/beast/http/parser.hpp

@@ -288,10 +288,10 @@ private:
         wr_.emplace(m_, content_length, ec);
     }
 
-    void
+    std::size_t
     on_data(string_view s, error_code& ec)
     {
-        wr_->put(boost::asio::buffer(
+        return wr_->put(boost::asio::buffer(
             s.data(), s.size()), ec);
     }
 

include/beast/http/string_body.hpp

@@ -105,7 +105,7 @@ struct string_body
         }
 
         template<class ConstBufferSequence>
-        void
+        std::size_t
         put(ConstBufferSequence const& buffers,
             error_code& ec)
         {
@@ -120,10 +120,10 @@ struct string_body
             catch(std::length_error const&)
             {
                 ec = error::buffer_overflow;
-                return;
+                return 0;
             }
             ec.assign(0, ec.category());
-            buffer_copy(boost::asio::buffer(
+            return buffer_copy(boost::asio::buffer(
                 &body_[0] + len, n), buffers);
         }
 

include/beast/http/type_traits.hpp

@@ -122,9 +122,10 @@ struct is_body_writer : std::false_type {};
 
 template<class T>
 struct is_body_writer<T, beast::detail::void_t<decltype(
-    std::declval<typename T::writer&>().put(
+    std::declval<std::size_t&>() =
-        std::declval<boost::asio::const_buffers_1>(),
+        std::declval<typename T::writer&>().put(
-        std::declval<error_code&>()),
+            std::declval<boost::asio::const_buffers_1>(),
+            std::declval<error_code&>()),
     std::declval<typename T::writer&>().finish(
         std::declval<error_code&>()),
     (void)0)>> : std::integral_constant<bool,

test/benchmarks/parser.cpp

@@ -199,11 +199,11 @@ public:
             ec.assign(0, ec.category());
         }
 
-        void
+        std::size_t
-        on_data(string_view,
+        on_data(string_view s, error_code& ec)
-            error_code& ec)
         {
             ec.assign(0, ec.category());
+            return s.size();
         }
 
         void

test/core/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Part of Beast
 
-GroupSources(examples examples)
+GroupSources(example example)
 GroupSources(extras/beast extras)
 GroupSources(include/beast beast)
 

test/exemplars.cpp

@@ -125,9 +125,11 @@ struct BodyWriter
         @param buffers The constant buffer sequence to store.
 
         @param ec Set to the error, if any occurred.
+
+        @return The number of bytes transferred from the input buffers.
     */
     template<class ConstBufferSequence>
-    void
+    std::size_t
     put(ConstBufferSequence const& buffers, error_code& ec)
     {
         // The specification requires this to indicate "no error"

test/http/doc_examples.cpp

@@ -49,13 +49,13 @@ BOOST_STATIC_ASSERT(::detail::is_mutable_container<string_view>::value == false)
 BOOST_STATIC_ASSERT(::detail::is_mutable_container<std::vector<char>>::value == true);
 BOOST_STATIC_ASSERT(::detail::is_mutable_container<std::list<char>>::value == false);
 
-class doc_http_samples_test
+class doc_examples_test
     : public beast::unit_test::suite
     , public beast::test::enable_yield_to
 {
 public:
     // two threads, for some examples using a pipe
-    doc_http_samples_test()
+    doc_examples_test()
         : enable_yield_to(2)
     {
     }
@@ -386,6 +386,27 @@ public:
 
     //--------------------------------------------------------------------------
 
+    void
+    doIncrementalRead()
+    {
+        test::pipe c{ios_};
+        std::string s(2048, '*');
+        ostream(c.server.buffer) <<
+            "HTTP/1.1 200 OK\r\n"
+            "Content-Length: 2048\r\n"
+            "Server: test\r\n"
+            "\r\n" <<
+            s;
+        error_code ec;
+        flat_buffer b;
+        std::stringstream ss;
+        read_and_print_body<false>(ss, c.server, b, ec);
+        if(BEAST_EXPECTS(! ec, ec.message()))
+            BEAST_EXPECT(ss.str() == s);
+    }
+
+    //--------------------------------------------------------------------------
+
     void
     run()
     {
@@ -399,10 +420,11 @@ public:
         doDeferredBody();
         doFileBody();
         doConstAndMutableBody();
+        doIncrementalRead();
     }
 };
 
-BEAST_DEFINE_TESTSUITE(doc_http_samples,http,beast);
+BEAST_DEFINE_TESTSUITE(doc_examples,http,beast);
 
 } // http
 } // beast

test/http/test_parser.hpp

@@ -118,7 +118,7 @@ public:
             ec.assign(0, ec.category());
     }
 
-    void
+    std::size_t
     on_data(string_view s,
         error_code& ec)
     {
@@ -127,6 +127,7 @@ public:
             fc_->fail(ec);
         else
             ec.assign(0, ec.category());
+        return s.size();
     }
 
     void