feedc0de/boost_beast
1
0
Fork 0
forked from boostorg/beast
Code Pull Requests Activity

Refactor chunked parsing (API Change):

Browse Source 
* parser now has a callback feature for intercepting chunk headers
  and chunk bodies

* The names for basic_parser derived class callbacks have been
  refactored

* basic_parser offers an additional callback for distinguishing
  chunk body data.

Actions Required:

* Adjust signatures for required members of user-defined
  subclasses of basic_parser

* Use the new basic_parser chunk callbacks for accessing
  chunk extensions and chunk bodies.
This commit is contained in:
Vinnie Falco
2017-07-20 08:15:31 -07:00
parent cc73d883be
commit 4b71aa67e0
27 changed files with 1851 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/benchmarks/parser.cpp
+18 -10
View File
@@ -166,14 +166,14 @@ public:
boost::asio::mutable_buffers_1;
void
on_request(verb, string_view,
on_request_impl(verb, string_view,
string_view, int, error_code& ec)
{
ec.assign(0, ec.category());
}
void
on_response(int,
on_response_impl(int,
string_view,
int, error_code& ec)
{
@@ -181,42 +181,50 @@ public:
}
void
on_field(field,
on_field_impl(field,
string_view, string_view, error_code& ec)
{
ec.assign(0, ec.category());
}
void
on_header(error_code& ec)
on_header_impl(error_code& ec)
{
ec.assign(0, ec.category());
}
void
on_body(boost::optional<std::uint64_t> const&,
on_body_init_impl(
boost::optional<std::uint64_t> const&,
error_code& ec)
{
ec.assign(0, ec.category());
}
std::size_t
on_data(string_view s, error_code& ec)
on_body_impl(string_view s, error_code& ec)
{
ec.assign(0, ec.category());
return s.size();
}
void
on_chunk(std::uint64_t,
string_view,
error_code& ec)
on_chunk_header_impl(std::uint64_t,
string_view, error_code& ec)
{
ec.assign(0, ec.category());
}
std::size_t
on_chunk_body_impl(std::uint64_t,
string_view s, error_code& ec)
{
ec.assign(0, ec.category());
return s.size();
}
void
on_complete(error_code& ec)
on_finish_impl(error_code& ec)
{
ec.assign(0, ec.category());
}
test/core/CMakeLists.txt
+1
View File
@@ -44,6 +44,7 @@ add_executable (core-tests
base64.cpp
empty_base_optimization.cpp
sha1.cpp
detail/varint.cpp
)
target_link_libraries(core-tests
test/core/Jamfile
+1
View File
@@ -38,4 +38,5 @@ unit-test core-tests :
base64.cpp
empty_base_optimization.cpp
sha1.cpp
detail/varint.cpp
;
test/core/detail/varint.cpp
+52
View File
@@ -0,0 +1,52 @@
//
// Copyright (c) 2013-2017 Vinnie Falco (vinnie dot falco at gmail dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Test that header file is self-contained.
#include <beast/core/detail/varint.hpp>
#include <beast/unit_test/suite.hpp>
namespace beast {
class varint_test : public beast::unit_test::suite
{
public:
void
testVarint()
{
using beast::detail::varint_read;
using beast::detail::varint_size;
using beast::detail::varint_write;
std::size_t n0 = 0;
std::size_t n1 = 1;
for(;;)
{
char buf[16];
BOOST_ASSERT(sizeof(buf) >= varint_size(n0));
auto it = &buf[0];
varint_write(it, n0);
it = &buf[0];
auto n = varint_read(it);
BEAST_EXPECT(n == n0);
n = n0 + n1;
if(n < n1)
break;
n0 = n1;
n1 = n;
}
}
void
run()
{
testVarint();
}
};
BEAST_DEFINE_TESTSUITE(varint,core,beast);
} // beast
test/http/basic_parser.cpp
+64 -4
View File
@@ -8,6 +8,7 @@
// Test that header file is self-contained.
#include <beast/http/basic_parser.hpp>
#include "message_fuzz.hpp"
#include "test_parser.hpp"
#include <beast/core/buffer_cat.hpp>
@@ -17,6 +18,7 @@
#include <beast/core/ostream.hpp>
#include <beast/http/parser.hpp>
#include <beast/http/string_body.hpp>
#include <beast/test/fuzz.hpp>
#include <beast/unit_test/suite.hpp>
namespace beast {
@@ -399,7 +401,7 @@ public:
BEAST_EXPECT(p.got_on_field == 2);
BEAST_EXPECT(p.got_on_header == 1);
BEAST_EXPECT(p.got_on_body == 1);
BEAST_EXPECT(p.got_on_chunk == 1);
BEAST_EXPECT(p.got_on_chunk == 2);
BEAST_EXPECT(p.got_on_complete == 1);
});
parsegrind<test_parser<false>>(
@@ -415,7 +417,7 @@ public:
BEAST_EXPECT(p.got_on_field == 2);
BEAST_EXPECT(p.got_on_header == 1);
BEAST_EXPECT(p.got_on_body == 1);
BEAST_EXPECT(p.got_on_chunk == 1);
BEAST_EXPECT(p.got_on_chunk == 2);
BEAST_EXPECT(p.got_on_complete == 1);
});
}
@@ -1104,7 +1106,64 @@ public:
//--------------------------------------------------------------------------
void
testFuzz1()
testFuzz()
{
auto const grind =
[&](string_view s)
{
static_string<100> ss{s};
test::fuzz_rand r;
test::fuzz(ss, 4, 5, r,
[&](string_view s)
{
error_code ec;
test_parser<false> p;
p.eager(true);
p.put(boost::asio::const_buffers_1{
s.data(), s.size()}, ec);
});
};
auto const good =
[&](string_view s)
{
std::string msg =
"HTTP/1.1 200 OK\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"0" + s.to_string() + "\r\n"
"\r\n";
error_code ec;
test_parser<false> p;
p.eager(true);
p.put(boost::asio::const_buffers_1{
msg.data(), msg.size()}, ec);
BEAST_EXPECTS(! ec, ec.message());
grind(msg);
};
auto const bad =
[&](string_view s)
{
std::string msg =
"HTTP/1.1 200 OK\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"0" + s.to_string() + "\r\n"
"\r\n";
error_code ec;
test_parser<false> p;
p.eager(true);
p.put(boost::asio::const_buffers_1{
msg.data(), msg.size()}, ec);
BEAST_EXPECT(ec);
grind(msg);
};
chunkExtensionsTest(good, bad);
}
//--------------------------------------------------------------------------
void
testRegression1()
{
// crash_00cda0b02d5166bd1039ddb3b12618cd80da75f3
unsigned char buf[] ={
@@ -1154,7 +1213,8 @@ public:
testIssue430();
testIssue452();
testIssue496();
testFuzz1();
testFuzz();
testRegression1();
}
};
test/http/chunk_encode.cpp
+83
View File
@@ -8,9 +8,14 @@
// Test that header file is self-contained.
#include <beast/http/chunk_encode.hpp>
#include "message_fuzz.hpp"
#include <beast/core/static_string.hpp>
#include <beast/http/fields.hpp>
#include <beast/test/fuzz.hpp>
#include <beast/unit_test/suite.hpp>
#include <boost/optional.hpp>
#include <random>
namespace beast {
namespace http {
@@ -194,17 +199,94 @@ public:
void
testChunkExtensions()
{
auto const str =
[](chunk_extensions const& ce)
{
std::string s;
for(auto const& v : ce)
{
s.append(v.first.to_string());
s.push_back(',');
if(! v.second.empty())
{
s.append(v.second.to_string());
s.push_back(',');
}
}
return s;
};
chunk_extensions ce;
ce.insert("x");
BEAST_EXPECT(ce.str() == ";x");
BEAST_EXPECT(str(ce) == "x,");
ce.insert("y", "z");
BEAST_EXPECT(ce.str() == ";x;y=z");
BEAST_EXPECT(str(ce) == "x,y,z,");
ce.insert("z", R"(")");
BEAST_EXPECT(ce.str() == R"(;x;y=z;z="\"")");
BEAST_EXPECT(str(ce) == R"(x,y,z,z,",)");
ce.insert("p", R"(\)");
BEAST_EXPECT(ce.str() == R"(;x;y=z;z="\"";p="\\")");
BEAST_EXPECT(str(ce) == R"(x,y,z,z,",p,\,)");
ce.insert("q", R"(1"2\)");
BEAST_EXPECT(ce.str() == R"(;x;y=z;z="\"";p="\\";q="1\"2\\")");
BEAST_EXPECT(str(ce) == R"(x,y,z,z,",p,\,q,1"2\,)");
}
void
testParseChunkExtensions()
{
auto const grind =
[&](string_view s)
{
error_code ec;
static_string<200> ss{s};
test::fuzz_rand r;
for(auto i = 3; i--;)
{
test::fuzz(ss, 5, 5, r,
[&](string_view s)
{
chunk_extensions c1;
c1.parse(s, ec);
if(ec)
{
pass();
return;
}
chunk_extensions c2;
c2.parse(c1.str(), ec);
if(! BEAST_EXPECTS(! ec, ec.message()))
return;
chunk_extensions c3;
for(auto const& v : c2)
if(v.second.empty())
c3.insert(v.first);
else
c3.insert(v.first, v.second);
BEAST_EXPECTS(c2.str() == c3.str(), c3.str());
});
}
};
auto const good =
[&](string_view s)
{
error_code ec;
chunk_extensions ce;
ce.parse(s, ec);
BEAST_EXPECTS(! ec, ec.message());
grind(s);
};
auto const bad =
[&](string_view s)
{
error_code ec;
chunk_extensions ce;
ce.parse(s, ec);
BEAST_EXPECT(ec);
grind(s);
};
chunkExtensionsTest(good, bad);
}
void
@@ -215,6 +297,7 @@ public:
testChunkBody();
testChunkFinal();
testChunkExtensions();
testParseChunkExtensions();
}
};
test/http/doc_examples.cpp
+19 -11
View File
@@ -22,8 +22,9 @@
#include <beast/unit_test/suite.hpp>
#include <sstream>
#include <array>
#include <vector>
#include <limits>
#include <list>
#include <vector>
namespace beast {
namespace http {
@@ -369,7 +370,7 @@ public:
ostream(c.client.buffer) <<
"HTTP/1.1 200 OK\r\n"
"Server: test\r\n"
"Accept: Expires, Content-MD5\r\n"
"Trailer: Expires, Content-MD5\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"5\r\n"
@@ -385,16 +386,23 @@ public:
"Content-MD5: f4a5c16584f03d90\r\n"
"\r\n";
flat_buffer b;
response_parser<empty_body> p;
read_header(c.client, b, p);
BOOST_ASSERT(p.is_chunked());
//while(! p.is_done())
{
// read the chunk header?
// read the next chunk?
}
error_code ec;
flat_buffer b;
std::stringstream ss;
print_chunked_body<false>(ss, c.client, b, ec);
BEAST_EXPECTS(! ec, ec.message());
BEAST_EXPECT(ss.str() ==
"Chunk Body: First\n"
"Extension: quality = 1.0\n"
"Chunk Body: Hello, world!\n"
"Extension: file = abc.txt\n"
"Extension: quality = 0.7\n"
"Chunk Body: The Next Chunk\n"
"Extension: last\n"
"Chunk Body: Last one\n"
"Expires: never\n"
"Content-MD5: f4a5c16584f03d90\n");
}
test/http/error.cpp
+2
View File
@@ -41,6 +41,7 @@ public:
check("beast.http", error::need_more);
check("beast.http", error::unexpected_body);
check("beast.http", error::need_buffer);
check("beast.http", error::end_of_chunk);
check("beast.http", error::buffer_overflow);
check("beast.http", error::body_limit);
check("beast.http", error::bad_alloc);
@@ -56,6 +57,7 @@ public:
check("beast.http", error::bad_content_length);
check("beast.http", error::bad_transfer_encoding);
check("beast.http", error::bad_chunk);
check("beast.http", error::bad_chunk_extension);
check("beast.http", error::bad_obs_fold);
}
};
test/http/message_fuzz.hpp
+36
View File
@@ -562,6 +562,42 @@ public:
using message_fuzz = message_fuzz_t<>;
template<class Good, class Bad>
void
chunkExtensionsTest(
Good const& good, Bad const& bad)
{
good("");
good(";x");
good(";x;y");
good(";x=y");
good(";x;y=z");
good(" ;x");
good("\t;x");
good(" \t;x");
good("\t ;x");
good(" ; x");
good(" ;\tx");
good("\t ; \tx");
good(";x= y");
good(" ;x= y");
good(" ; x= y");
good(R"(;x="\"")");
good(R"(;x="\\")");
good(R"(;x;y=z;z="\"";p="\\";q="1\"2\\")");
bad(" ");
bad(";");
bad("=");
bad(" ;");
bad("; ");
bad(" ; ");
bad(" ; x ");
bad(";x =");
bad(";x = ");
bad(";x==");
}
} // http
} // beast
test/http/parser.cpp
-44
View File
@@ -328,49 +328,6 @@ public:
BEAST_EXPECT(used == 0);
}
void
testCallback()
{
{
multi_buffer b;
ostream(b) <<
"POST / HTTP/1.1\r\n"
"Content-Length: 2\r\n"
"\r\n"
"**";
error_code ec;
parser<true, string_body> p;
p.eager(true);
p.put(b.data(), ec);
p.on_header(
[this](parser<true, string_body>& p, error_code& ec)
{
BEAST_EXPECT(p.is_header_done());
ec.assign(0, ec.category());
});
BEAST_EXPECTS(! ec, ec.message());
}
{
multi_buffer b;
ostream(b) <<
"POST / HTTP/1.1\r\n"
"Content-Length: 2\r\n"
"\r\n"
"**";
error_code ec;
parser<true, string_body> p;
p.eager(true);
p.put(b.data(), ec);
p.on_header(
[this](parser<true, string_body>&, error_code& ec)
{
ec.assign(errc::bad_message,
generic_category());
});
BEAST_EXPECTS(! ec, ec.message());
}
}
void
run() override
{
@@ -378,7 +335,6 @@ public:
testNeedMore<flat_buffer>();
testNeedMore<multi_buffer>();
testGotSome();
testCallback();
}
};
test/http/test_parser.hpp
+28 -11
View File
@@ -51,7 +51,7 @@ public:
}
void
on_request(verb, string_view method_str_,
on_request_impl(verb, string_view method_str_,
string_view path_, int version_, error_code& ec)
{
method = std::string(
@@ -67,7 +67,7 @@ public:
}
void
on_response(int code,
on_response_impl(int code,
string_view reason_,
int version_, error_code& ec)
{
@@ -83,7 +83,7 @@ public:
}
void
on_field(field, string_view name,
on_field_impl(field, string_view name,
string_view value, error_code& ec)
{
++got_on_field;
@@ -95,7 +95,7 @@ public:
}
void
on_header(error_code& ec)
on_header_impl(error_code& ec)
{
++got_on_header;
if(fc_)
@@ -105,9 +105,9 @@ public:
}
void
on_body(boost::optional<
std::uint64_t> const& content_length_,
error_code& ec)
on_body_init_impl(
boost::optional<std::uint64_t> const& content_length_,
error_code& ec)
{
++got_on_body;
got_content_length =
@@ -119,7 +119,7 @@ public:
}
std::size_t
on_data(string_view s,
on_body_impl(string_view s,
error_code& ec)
{
body.append(s.data(), s.size());
@@ -131,8 +131,10 @@ public:
}
void
on_chunk(std::uint64_t,
string_view, error_code& ec)
on_chunk_header_impl(
std::uint64_t,
string_view,
error_code& ec)
{
++got_on_chunk;
if(fc_)
@@ -141,8 +143,23 @@ public:
ec.assign(0, ec.category());
}
std::size_t
on_chunk_body_impl(
std::uint64_t,
string_view s,
error_code& ec)
{
body.append(s.data(), s.size());
if(fc_)
fc_->fail(ec);
else
ec.assign(0, ec.category());
return s.size();
}
void
on_complete(error_code& ec)
on_finish_impl(error_code& ec)
{
++got_on_complete;
if(fc_)