feedc0de/boost_beast
1
0 Fork 0
forked from boostorg/beast
Code Pull Requests Activity

Refactor chunked-encoding serialization:

Browse Source 
New buffer sequence classes are provided to allow full
control over the serialization of chunk-encoded message
payloads:

* chunk_header

    A ConstBufferSequence representing the chunk header.
    It includes a hexadecimal-encoded size, an optional
    set of chunk extensions, and the trailing CRLF
    required to denote the end of the chunk header.

    This allows the caller to manually serialize the chunk
    body in one or more calls to a stream output function.
    The caller must also output an object of type `chunk_crlf`
    to the stream to indicate the end of the chunk body.

* chunk_crlf

    A small ConstBufferSequence that simply represents
    the two character sequence "\r\n" (CRLF). This is needed
    for the case where the caller wants to output a chunk
    body as a series of buffers (i.e. "chunking a chunk").

* chunk_body

    A ConstBufferSequence representing a complete chunk.
    This includes the size, an optional set of chunk extensions,
    a caller provided buffer containing the body, and the
    required CRLF that follows.

* chunk_final

    A ConstBufferSequence representing a final chunk. It
    includes an optional set of caller-provided field trailers

* chunk_extensions

    A container for building a set of chunk extensions to use
    during serialization. The use of the container is optional,
    callers may provide their own buffer containing a correctly
    formatted set of chunk extensions, or they may use their
    own convenience container which meets the requirements.

The basic_fields container is modified to allow construction
outside the context of a message. The container can be used
to provide trailers to `chunk_final`.

Actions Required:

* Remove references to ChunkDecorators. Use the new chunk-encoding
  buffer sequences to manually produce a chunked payload body in
  the case where control over the chunk-extensions and/or trailers
  is required.
This commit is contained in:
Vinnie Falco committed 2017-07-20 08:15:31 -07:00
1 parent 02de2e5cbb
commit ff15cf8688
69 files changed
+2312 -686
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/http/CMakeLists.txt
+1
View File
@@ -17,6 +17,7 @@ add_executable (http-tests
basic_file_body.cpp
basic_parser.cpp
buffer_body.cpp
chunk_encode.cpp
doc_examples.cpp
doc_snippets.cpp
dynamic_body.cpp
test/http/Jamfile
+1
View File
@@ -11,6 +11,7 @@ unit-test http-tests :
basic_file_body.cpp
basic_parser.cpp
buffer_body.cpp
chunk_encode.cpp
doc_examples.cpp
doc_snippets.cpp
dynamic_body.cpp
test/http/chunk_encode.cpp
+225
View File
@@ -0,0 +1,225 @@
//
// Copyright (c) 2013-2017 Vinnie Falco (vinnie dot falco at gmail dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Test that header file is self-contained.
#include <beast/http/chunk_encode.hpp>
#include <beast/http/fields.hpp>
#include <beast/unit_test/suite.hpp>
#include <boost/optional.hpp>
namespace beast {
namespace http {
class chunk_encode_test
: public beast::unit_test::suite
{
public:
struct not_chunk_extensions {};
BOOST_STATIC_ASSERT(
detail::is_chunk_extensions<chunk_extensions>::value);
BOOST_STATIC_ASSERT(
! detail::is_chunk_extensions<not_chunk_extensions>::value);
template<class ConstBufferSequence>
static
std::string
to_string(ConstBufferSequence const& buffers)
{
std::string s;
s.reserve(boost::asio::buffer_size(buffers));
for(boost::asio::const_buffer buffer : buffers)
s.append(
boost::asio::buffer_cast<char const*>(buffer),
boost::asio::buffer_size(buffer));
return s;
}
template<class T, class... Args>
void
check(string_view match, Args&&... args)
{
T t{std::forward<Args>(args)...};
BEAST_EXPECT(to_string(t) == match);
T t2{t};
BEAST_EXPECT(to_string(t2) == match);
T t3{std::move(t2)};
BEAST_EXPECT(to_string(t3) == match);
}
template<class T, class... Args>
void
check_fwd(string_view match, Args&&... args)
{
T t{std::forward<Args>(args)...};
BEAST_EXPECT(to_string(t) == match);
T t2{t};
BEAST_EXPECT(to_string(t2) == match);
T t3{std::move(t2)};
BEAST_EXPECT(to_string(t3) == match);
}
using cb_t = boost::asio::const_buffers_1;
static
cb_t
cb(string_view s)
{
return {s.data(), s.size()};
}
void
testChunkCRLF()
{
#if ! defined(BOOST_GCC) || BOOST_GCC >= 50000
check<chunk_crlf>("\r\n");
#endif
}
void
testChunkHeader()
{
check<chunk_header>("10\r\n", 16u);
check<chunk_header>("20;x\r\n", 32u, ";x");
chunk_extensions exts;
exts.insert("y");
exts.insert("z");
check<chunk_header>("30;y;z\r\n", 48u, exts);
{
auto exts2 = exts;
check_fwd<chunk_header>(
"30;y;z\r\n", 48u, std::move(exts2));
}
check<chunk_header>("30;y;z\r\n", 48u, exts,
std::allocator<double>{});
{
auto exts2 = exts;
check<chunk_header>(
"30;y;z\r\n", 48u, std::move(exts2),
std::allocator<double>{});
}
}
void
testChunkBody()
{
check<chunk_body<cb_t>>(
"3\r\n***\r\n", cb("***"));
check<chunk_body<cb_t>>(
"3;x\r\n***\r\n", cb("***"), ";x");
chunk_extensions exts;
exts.insert("y");
exts.insert("z");
check<chunk_body<cb_t>>(
"3;y;z\r\n***\r\n",
cb("***"), exts);
{
auto exts2 = exts;
check_fwd<chunk_body<cb_t>>(
"3;y;z\r\n***\r\n",
cb("***"), std::move(exts2));
}
check<chunk_body<cb_t>>(
"3;y;z\r\n***\r\n",
cb("***"), exts, std::allocator<double>{});
{
auto exts2 = exts;
check_fwd<chunk_body<cb_t>>(
"3;y;z\r\n***\r\n",
cb("***"), std::move(exts2),
std::allocator<double>{});
}
}
void
testChunkFinal()
{
check<chunk_last<>>(
"0\r\n\r\n");
check<chunk_last<cb_t>>(
"0\r\nMD5:ou812\r\n\r\n",
cb("MD5:ou812\r\n\r\n"));
fields trailers;
trailers.set(field::content_md5, "ou812");
check<chunk_last<fields>>(
"0\r\nContent-MD5: ou812\r\n\r\n",
trailers);
{
auto trailers2 = trailers;
check_fwd<chunk_last<fields>>(
"0\r\nContent-MD5: ou812\r\n\r\n",
std::move(trailers2));
}
check<chunk_last<fields>>(
"0\r\nContent-MD5: ou812\r\n\r\n",
trailers, std::allocator<double>{});
{
auto trailers2 = trailers;
check<chunk_last<fields>>(
"0\r\nContent-MD5: ou812\r\n\r\n",
std::move(trailers2), std::allocator<double>{});
}
}
void
testChunkExtensions()
{
chunk_extensions ce;
ce.insert("x");
BEAST_EXPECT(ce.str() == ";x");
ce.insert("y", "z");
BEAST_EXPECT(ce.str() == ";x;y=z");
ce.insert("z", R"(")");
BEAST_EXPECT(ce.str() == R"(;x;y=z;z="\"")");
ce.insert("p", R"(\)");
BEAST_EXPECT(ce.str() == R"(;x;y=z;z="\"";p="\\")");
ce.insert("q", R"(1"2\)");
BEAST_EXPECT(ce.str() == R"(;x;y=z;z="\"";p="\\";q="1\"2\\")");
}
void
run() override
{
testChunkCRLF();
testChunkHeader();
testChunkBody();
testChunkFinal();
testChunkExtensions();
}
};
BEAST_DEFINE_TESTSUITE(chunk_encode,http,beast);
} // http
} // beast
test/http/doc_examples.cpp
+120
View File
@@ -7,8 +7,14 @@
#include "example/doc/http_examples.hpp"
#include <beast/core/flat_buffer.hpp>
#include <beast/core/read_size.hpp>
#include <beast/core/ostream.hpp>
#include <beast/core/detail/clamp.hpp>
#include <beast/http/chunk_encode.hpp>
#include <beast/http/parser.hpp>
#include <beast/http/read.hpp>
#include <beast/http/write.hpp>
#include <beast/test/pipe_stream.hpp>
#include <beast/test/string_istream.hpp>
#include <beast/test/string_ostream.hpp>
@@ -282,6 +288,118 @@ public:
//--------------------------------------------------------------------------
void
doExplicitChunkSerialize()
{
auto const buf =
[](string_view s)
{
return boost::asio::const_buffers_1{
s.data(), s.size()};
};
test::pipe p{ios_};
response<empty_body> res{status::ok, 11};
res.set(field::server, "test");
res.set(field::accept, "Expires, Content-MD5");
res.chunked(true);
error_code ec;
response_serializer<empty_body> sr{res};
write_header(p.client, sr, ec);
chunk_extensions exts;
boost::asio::write(p.client,
make_chunk(buf("First")), ec);
exts.insert("quality", "1.0");
boost::asio::write(p.client,
make_chunk(buf("Hello, world!"), exts), ec);
exts.clear();
exts.insert("file", "abc.txt");
exts.insert("quality", "0.7");
boost::asio::write(p.client,
make_chunk(buf("The Next Chunk"), std::move(exts)), ec);
exts.clear();
exts.insert("last");
boost::asio::write(p.client,
make_chunk(buf("Last one"), std::move(exts),
std::allocator<double>{}), ec);
fields trailers;
trailers.set(field::expires, "never");
trailers.set(field::content_md5, "f4a5c16584f03d90");
boost::asio::write(p.client,
make_chunk_last(
trailers,
std::allocator<double>{}
), ec);
BEAST_EXPECT(
boost::lexical_cast<std::string>(
buffers(p.server.buffer.data())) ==
"HTTP/1.1 200 OK\r\n"
"Server: test\r\n"
"Accept: Expires, Content-MD5\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"5\r\n"
"First\r\n"
"d;quality=1.0\r\n"
"Hello, world!\r\n"
"e;file=abc.txt;quality=0.7\r\n"
"The Next Chunk\r\n"
"8;last\r\n"
"Last one\r\n"
"0\r\n"
"Expires: never\r\n"
"Content-MD5: f4a5c16584f03d90\r\n"
"\r\n");
}
//--------------------------------------------------------------------------
void
doExplicitChunkParse()
{
test::pipe c{ios_};
ostream(c.client.buffer) <<
"HTTP/1.1 200 OK\r\n"
"Server: test\r\n"
"Accept: Expires, Content-MD5\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"5\r\n"
"First\r\n"
"d;quality=1.0\r\n"
"Hello, world!\r\n"
"e;file=abc.txt;quality=0.7\r\n"
"The Next Chunk\r\n"
"8;last\r\n"
"Last one\r\n"
"0\r\n"
"Expires: never\r\n"
"Content-MD5: f4a5c16584f03d90\r\n"
"\r\n";
flat_buffer b;
response_parser<empty_body> p;
read_header(c.client, b, p);
BOOST_ASSERT(p.is_chunked());
//while(! p.is_done())
{
// read the chunk header?
// read the next chunk?
}
}
//--------------------------------------------------------------------------
void
run()
{
@@ -294,6 +412,8 @@ public:
doHEAD();
doDeferredBody();
doIncrementalRead();
doExplicitChunkSerialize();
doExplicitChunkParse();
}
};
test/http/doc_snippets.cpp
+173 -22
View File
@@ -22,6 +22,16 @@ using namespace beast::http;
namespace doc_http_snippets {
//[http_snippet_17
// This function returns the buffer containing the next chunk body
boost::asio::const_buffers_1 get_next_chunk_body();
//]
boost::asio::const_buffers_1 get_next_chunk_body()
{
return {nullptr, 0};
}
void fxx() {
boost::asio::io_service ios;
@@ -130,8 +140,170 @@ void fxx() {
//]
}
{
//[http_snippet_18
// Prepare an HTTP/1.1 response with a chunked body
response<empty_body> res{status::ok, 11};
res.set(field::server, "Beast");
// Set Transfer-Encoding to "chunked".
// If a Content-Length was present, it is removed.
res.chunked(true);
// Set up the serializer
response_serializer<empty_body> sr{res};
// Write the header first
write_header(sock, sr);
// Now manually emit three chunks:
boost::asio::write(sock, make_chunk(get_next_chunk_body()));
boost::asio::write(sock, make_chunk(get_next_chunk_body()));
boost::asio::write(sock, make_chunk(get_next_chunk_body()));
// We are responsible for sending the last chunk:
boost::asio::write(sock, make_chunk_last());
//]
}
{
//[http_snippet_19
// Prepare a set of chunk extension to emit with the body
chunk_extensions ext;
ext.insert("mp3");
ext.insert("title", "Beale Street Blues");
ext.insert("artist", "W.C. Handy");
// Write the next chunk with the chunk extensions
// The implementation will make a copy of the extensions object,
// so the caller does not need to manage lifetime issues.
boost::asio::async_write(sock, make_chunk(get_next_chunk_body(), ext),
[](error_code ec, std::size_t)
{
if(ec)
std::cout << "Error: " << ec.message() << std::endl;
});
// Write the next chunk with the chunk extensions
// The implementation will make a copy of the extensions object, storing the copy
// using the custom allocator, so the caller does not need to manage lifetime issues.
boost::asio::async_write(sock,
make_chunk(get_next_chunk_body(), ext, std::allocator<char>{}),
[](error_code ec, std::size_t)
{
if(ec)
std::cout << "Error: " << ec.message() << std::endl;
});
// Write the next chunk with the chunk extensions
// The implementation allocates memory using the default allocator and takes ownership
// of the extensions object, so the caller does not need to manage lifetime issues.
// Note: ext is moved
boost::asio::async_write(sock, make_chunk(get_next_chunk_body(), std::move(ext)),
[](error_code ec, std::size_t)
{
if(ec)
std::cout << "Error: " << ec.message() << std::endl;
});
//]
}
{
//[http_snippet_20
// Manually specify the chunk extensions.
// Some of the strings contain spaces and a period and must be quoted
boost::asio::write(sock, make_chunk(get_next_chunk_body(),
";mp3"
";title=\"Danny Boy\""
";artist=\"Fred E. Weatherly\""
));
//]
}
{
//[http_snippet_21
// Prepare a chunked HTTP/1.1 response with some trailer fields
response<empty_body> res{status::ok, 11};
res.set(field::server, "Beast");
// Inform the client of the trailer fields we will send
res.set(field::trailer, "Content-MD5, Expires");
res.chunked(true);
// Serialize the header and two chunks
response_serializer<empty_body> sr{res};
write_header(sock, sr);
boost::asio::write(sock, make_chunk(get_next_chunk_body()));
boost::asio::write(sock, make_chunk(get_next_chunk_body()));
// Prepare the trailer
fields trailer;
trailer.set(field::content_md5, "f4a5c16584f03d90");
trailer.set(field::expires, "never");
// Emit the trailer in the last chunk.
// The implementation will use the default allocator to create the storage for holding
// the serialized fields.
boost::asio::write(sock, make_chunk_last(trailer));
//]
}
{
//[http_snippet_22
// Use a custom allocator for serializing the last chunk
fields trailer;
trailer.set(field::approved, "yes");
boost::asio::write(sock, make_chunk_last(trailer, std::allocator<char>{}));
//]
}
{
//[http_snippet_23
// Manually emit a trailer.
// We are responsible for ensuring that the trailer format adheres to the specification.
string_view ext =
"Content-MD5: f4a5c16584f03d90\r\n"
"Expires: never\r\n"
"\r\n";
boost::asio::write(sock, make_chunk_last(boost::asio::const_buffers_1{ext.data(), ext.size()}));
//]
}
{
//[http_snippet_24
// Prepare a chunked HTTP/1.1 response and send the header
response<empty_body> res{status::ok, 11};
res.set(field::server, "Beast");
res.chunked(true);
response_serializer<empty_body> sr{res};
write_header(sock, sr);
// Obtain three body buffers up front
auto const cb1 = get_next_chunk_body();
auto const cb2 = get_next_chunk_body();
auto const cb3 = get_next_chunk_body();
// Manually emit a chunk by first writing the chunk-size header with the correct size
boost::asio::write(sock, chunk_header{
boost::asio::buffer_size(cb1) +
boost::asio::buffer_size(cb2) +
boost::asio::buffer_size(cb3)});
// And then output the chunk body in three pieces ("chunk the chunk")
boost::asio::write(sock, cb1);
boost::asio::write(sock, cb2);
boost::asio::write(sock, cb3);
// When we go this deep, we are also responsible for the terminating CRLF
boost::asio::write(sock, chunk_crlf{});
//]
}
} // fxx()
//[http_snippet_12
/** Send a message to a stream synchronously.
@@ -302,27 +474,6 @@ split_print_cxx14(message<isRequest, Body, Fields> const& m)
//]
#endif
//[http_snippet_17
struct decorator
{
std::string s;
template<class ConstBufferSequence>
string_view
operator()(ConstBufferSequence const& buffers)
{
s = ";x=" + std::to_string(boost::asio::buffer_size(buffers));
return s;
}
string_view
operator()(boost::asio::null_buffers)
{
return "Result: OK\r\n";
}
};
//]
// Highest snippet:
} // doc_http_snippets
test/http/write.cpp
+16 -49
View File
@@ -629,15 +629,16 @@ public:
}
}
template<class Stream,
bool isRequest, class Body, class Fields,
class Decorator = no_chunk_decorator>
template<
class Stream,
bool isRequest, class Body, class Fields>
void
do_write(Stream& stream, message<
isRequest, Body, Fields> const& m, error_code& ec,
Decorator const& decorator = Decorator{})
do_write(
Stream& stream,
message<isRequest, Body, Fields> const& m,
error_code& ec)
{
serializer<isRequest, Body, Fields, Decorator> sr{m, decorator};
serializer<isRequest, Body, Fields> sr{m};
for(;;)
{
stream.nwrite = 0;
@@ -650,16 +651,17 @@ public:
}
}
template<class Stream,
bool isRequest, class Body, class Fields,
class Decorator = no_chunk_decorator>
template<
class Stream,
bool isRequest, class Body, class Fields>
void
do_async_write(Stream& stream,
do_async_write(
Stream& stream,
message<isRequest, Body, Fields> const& m,
error_code& ec, yield_context yield,
Decorator const& decorator = Decorator{})
error_code& ec,
yield_context yield)
{
serializer<isRequest, Body, Fields, Decorator> sr{m, decorator};
serializer<isRequest, Body, Fields> sr{m};
for(;;)
{
stream.nwrite = 0;
@@ -672,25 +674,6 @@ public:
}
}
struct test_decorator
{
std::string s;
template<class ConstBufferSequence>
string_view
operator()(ConstBufferSequence const& buffers)
{
s = ";x=" + std::to_string(boost::asio::buffer_size(buffers));
return s;
}
string_view
operator()(boost::asio::null_buffers)
{
return "Result: OK\r\n";
}
};
template<class Body>
void
testWriteStream(boost::asio::yield_context yield)
@@ -768,14 +751,6 @@ public:
p.server.str(), m.body.s));
p.server.clear();
}
{
auto m = m0;
error_code ec;
do_write(p.client, m, ec, test_decorator{});
BEAST_EXPECT(equal_body<false>(
p.server.str(), m.body.s));
p.server.clear();
}
{
auto m = m0;
error_code ec;
@@ -784,14 +759,6 @@ public:
p.server.str(), m.body.s));
p.server.clear();
}
{
auto m = m0;
error_code ec;
do_async_write(p.client, m, ec, yield, test_decorator{});
BEAST_EXPECT(equal_body<false>(
p.server.str(), m.body.s));
p.server.clear();
}
{
auto m = m0;
error_code ec;