feedc0de/boost_beast
1
0
Fork 0
forked from boostorg/beast
Code Pull Requests Activity

Faster zlib tests

Browse Source
This commit is contained in:
Vinnie Falco
2017-07-31 18:40:14 -07:00
parent b0d7bc2e86
commit 7b90ff852d
4 changed files with 235 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -7,6 +7,7 @@ Version 96:
* Fix library.json
* Update boostorg links
* Add bench-zlib
* Faster zlib tests
--------------------------------------------------------------------------------

294
test/beast/zlib/deflate_stream.cpp
View File

@@ -10,9 +10,14 @@
// Test that header file is self-contained.
#include <boost/beast/zlib/deflate_stream.hpp>
#include "ztest.hpp"
#include <boost/beast/core/string.hpp>
#include <boost/beast/unit_test/suite.hpp>
#include <cstdint>
#include <random>
#include "zlib-1.2.11/zlib.h"
#include "ztest.hpp"
namespace boost {
namespace beast {
@@ -21,6 +26,127 @@ namespace zlib {
class deflate_stream_test : public beast::unit_test::suite
{
public:
// Lots of repeats, limited char range
static
std::string
corpus1(std::size_t n)
{
static std::string const alphabet{
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
};
std::string s;
s.reserve(n + 5);
std::mt19937 g;
std::uniform_int_distribution<std::size_t> d0{
0, alphabet.size() - 1};
std::uniform_int_distribution<std::size_t> d1{
1, 5};
while(s.size() < n)
{
auto const rep = d1(g);
auto const ch = alphabet[d0(g)];
s.insert(s.end(), rep, ch);
}
s.resize(n);
return s;
}
// Random data
static
std::string
corpus2(std::size_t n)
{
std::string s;
s.reserve(n);
std::mt19937 g;
std::uniform_int_distribution<std::uint32_t> d0{0, 255};
while(n--)
s.push_back(static_cast<char>(d0(g)));
return s;
}
static
std::string
compress(
string_view const& in,
int level, // 0=none, 1..9, -1=default
int windowBits, // 9..15
int memLevel) // 1..9 (8=default)
{
int const strategy = Z_DEFAULT_STRATEGY;
int result;
z_stream zs;
memset(&zs, 0, sizeof(zs));
result = deflateInit2(
&zs,
level,
Z_DEFLATED,
-windowBits,
memLevel,
strategy);
if(result != Z_OK)
throw std::logic_error{"deflateInit2 failed"};
zs.next_in = (Bytef*)in.data();
zs.avail_in = static_cast<uInt>(in.size());
std::string out;
out.resize(deflateBound(&zs,
static_cast<uLong>(in.size())));
zs.next_in = (Bytef*)in.data();
zs.avail_in = static_cast<uInt>(in.size());
zs.next_out = (Bytef*)&out[0];
zs.avail_out = static_cast<uInt>(out.size());
result = deflate(&zs, Z_FULL_FLUSH);
if(result != Z_OK)
throw std::logic_error("deflate failed");
out.resize(zs.total_out);
deflateEnd(&zs);
return out;
}
static
std::string
decompress(string_view const& in)
{
int result;
std::string out;
z_stream zs;
memset(&zs, 0, sizeof(zs));
result = inflateInit2(&zs, -15);
try
{
zs.next_in = (Bytef*)in.data();
zs.avail_in = static_cast<uInt>(in.size());
for(;;)
{
out.resize(zs.total_out + 1024);
zs.next_out = (Bytef*)&out[zs.total_out];
zs.avail_out = static_cast<uInt>(
out.size() - zs.total_out);
result = inflate(&zs, Z_SYNC_FLUSH);
if( result == Z_NEED_DICT ||
result == Z_DATA_ERROR ||
result == Z_MEM_ERROR)
{
throw std::logic_error("inflate failed");
}
if(zs.avail_out > 0)
break;
if(result == Z_STREAM_END)
break;
}
out.resize(zs.total_out);
inflateEnd(&zs);
}
catch(...)
{
inflateEnd(&zs);
throw;
}
return out;
}
//--------------------------------------------------------------------------
using self = deflate_stream_test;
typedef void(self::*pmf_t)(
int level, int windowBits, int strategy,
@@ -41,55 +167,6 @@ public:
}
}
void
doDeflate1_zlib(
int level, int windowBits, int strategy,
std::string const& check)
{
int result;
std::string out;
::z_stream zs;
std::memset(&zs, 0, sizeof(zs));
result = deflateInit2(&zs,
level,
Z_DEFLATED,
-windowBits,
8,
strategy);
if(! BEAST_EXPECT(result == Z_OK))
goto err;
out.resize(deflateBound(&zs,
static_cast<uLong>(check.size())));
zs.next_in = (Bytef*)check.data();
zs.avail_in = static_cast<uInt>(check.size());
zs.next_out = (Bytef*)out.data();
zs.avail_out = static_cast<uInt>(out.size());
{
bool progress = true;
for(;;)
{
result = deflate(&zs, Z_FULL_FLUSH);
if( result == Z_BUF_ERROR ||
result == Z_STREAM_END) // per zlib FAQ
goto fin;
if(! BEAST_EXPECT(progress))
goto err;
progress = false;
}
}
fin:
out.resize(zs.total_out);
{
z_inflator zi;
auto const s = zi(out);
BEAST_EXPECT(s == check);
}
err:
deflateEnd(&zs);
}
void
doDeflate1_beast(
int level, int windowBits, int strategy,
@@ -128,11 +205,7 @@ public:
fin:
out.resize(zs.total_out);
{
z_inflator zi;
auto const s = zi(out);
BEAST_EXPECT(s == check);
}
BEAST_EXPECT(decompress(out) == check);
err:
;
@@ -140,77 +213,6 @@ public:
//--------------------------------------------------------------------------
void
doDeflate2_zlib(
int level, int windowBits, int strategy,
std::string const& check)
{
for(std::size_t i = 1; i < check.size(); ++i)
{
for(std::size_t j = 1;; ++j)
{
int result;
::z_stream zs;
std::memset(&zs, 0, sizeof(zs));
result = deflateInit2(&zs,
level,
Z_DEFLATED,
-windowBits,
8,
strategy);
if(! BEAST_EXPECT(result == Z_OK))
continue;
std::string out;
out.resize(deflateBound(&zs,
static_cast<uLong>(check.size())));
if(j >= out.size())
{
deflateEnd(&zs);
break;
}
zs.next_in = (Bytef*)check.data();
zs.avail_in = static_cast<uInt>(i);
zs.next_out = (Bytef*)out.data();
zs.avail_out = static_cast<uInt>(j);
bool bi = false;
bool bo = false;
for(;;)
{
int flush = bi ? Z_FULL_FLUSH : Z_NO_FLUSH;
result = deflate(&zs, flush);
if( result == Z_BUF_ERROR ||
result == Z_STREAM_END) // per zlib FAQ
goto fin;
if(! BEAST_EXPECT(result == Z_OK))
goto err;
if(zs.avail_in == 0 && ! bi)
{
bi = true;
zs.avail_in =
static_cast<uInt>(check.size() - i);
}
if(zs.avail_out == 0 && ! bo)
{
bo = true;
zs.avail_out =
static_cast<uInt>(out.size() - j);
}
}
fin:
out.resize(zs.total_out);
{
z_inflator zi;
auto const s = zi(out);
BEAST_EXPECT(s == check);
}
err:
deflateEnd(&zs);
}
}
}
void
doDeflate2_beast(
int level, int windowBits, int strategy,
@@ -264,11 +266,7 @@ public:
fin:
out.resize(zs.total_out);
{
z_inflator zi;
auto const s = zi(out);
BEAST_EXPECT(s == check);
}
BEAST_EXPECT(decompress(out) == check);
err:
;
@@ -279,12 +277,8 @@ public:
//--------------------------------------------------------------------------
void
doMatrix(std::string const& label,
std::string const& check, pmf_t pmf)
doMatrix(std::string const& check, pmf_t pmf)
{
using namespace std::chrono;
using clock_type = steady_clock;
auto const when = clock_type::now();
for(int level = 0; level <= 9; ++level)
{
for(int windowBits = 8; windowBits <= 9; ++windowBits)
@@ -299,33 +293,15 @@ public:
}
}
}
auto const elapsed = clock_type::now() - when;
log <<
label << ": " <<
duration_cast<
milliseconds>(elapsed).count() << "ms\n";
log.flush();
}
void
testDeflate()
{
doMatrix("1.beast ", "Hello, world!", &self::doDeflate1_beast);
doMatrix("1.zlib ", "Hello, world!", &self::doDeflate1_zlib);
#if ! BOOST_BEAST_NO_SLOW_TESTS
doMatrix("2.beast ", "Hello, world!", &self::doDeflate2_beast);
doMatrix("2.zlib ", "Hello, world!", &self::doDeflate2_zlib);
{
auto const s = corpus1(56);
doMatrix("3.beast ", s, &self::doDeflate2_beast);
doMatrix("3.zlib ", s, &self::doDeflate2_zlib);
}
{
auto const s = corpus1(512 * 1024);
doMatrix("4.beast ", s, &self::doDeflate1_beast);
doMatrix("4.zlib ", s, &self::doDeflate1_zlib);
}
#endif
doMatrix("Hello, world!", &self::doDeflate1_beast);
doMatrix("Hello, world!", &self::doDeflate2_beast);
doMatrix(corpus1(56), &self::doDeflate2_beast);
doMatrix(corpus1(1024), &self::doDeflate1_beast);
}
void

239
test/beast/zlib/inflate_stream.cpp
View File

@@ -10,12 +10,13 @@
// Test that header file is self-contained.
#include <boost/beast/zlib/inflate_stream.hpp>
#include "ztest.hpp"
#include <boost/beast/core/string.hpp>
#include <boost/beast/unit_test/suite.hpp>
#include <chrono>
#include <random>
#include "ztest.hpp"
namespace boost {
namespace beast {
namespace zlib {
@@ -23,6 +24,83 @@ namespace zlib {
class inflate_stream_test : public beast::unit_test::suite
{
public:
// Lots of repeats, limited char range
static
std::string
corpus1(std::size_t n)
{
static std::string const alphabet{
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
};
std::string s;
s.reserve(n + 5);
std::mt19937 g;
std::uniform_int_distribution<std::size_t> d0{
0, alphabet.size() - 1};
std::uniform_int_distribution<std::size_t> d1{
1, 5};
while(s.size() < n)
{
auto const rep = d1(g);
auto const ch = alphabet[d0(g)];
s.insert(s.end(), rep, ch);
}
s.resize(n);
return s;
}
// Random data
static
std::string
corpus2(std::size_t n)
{
std::string s;
s.reserve(n);
std::mt19937 g;
std::uniform_int_distribution<std::uint32_t> d0{0, 255};
while(n--)
s.push_back(static_cast<char>(d0(g)));
return s;
}
static
std::string
compress(
string_view const& in,
int level, // 0=none, 1..9, -1=default
int windowBits, // 9..15
int memLevel, // 1..9 (8=default)
int strategy) // e.g. Z_DEFAULT_STRATEGY
{
int result;
z_stream zs;
memset(&zs, 0, sizeof(zs));
result = deflateInit2(
&zs,
level,
Z_DEFLATED,
-windowBits,
memLevel,
strategy);
if(result != Z_OK)
throw std::logic_error{"deflateInit2 failed"};
zs.next_in = (Bytef*)in.data();
zs.avail_in = static_cast<uInt>(in.size());
std::string out;
out.resize(deflateBound(&zs,
static_cast<uLong>(in.size())));
zs.next_in = (Bytef*)in.data();
zs.avail_in = static_cast<uInt>(in.size());
zs.next_out = (Bytef*)&out[0];
zs.avail_out = static_cast<uInt>(out.size());
result = deflate(&zs, Z_FULL_FLUSH);
if(result != Z_OK)
throw std::logic_error("deflate failed");
out.resize(zs.total_out);
deflateEnd(&zs);
return out;
}
//--------------------------------------------------------------------------
enum Split
@@ -120,100 +198,6 @@ public:
}
};
class ZLib
{
Split in_;
Split check_;
int flush_;
public:
ZLib(Split in, Split check, int flush = Z_SYNC_FLUSH)
: in_(in)
, check_(check)
, flush_(flush)
{
}
void
operator()(
int window,
std::string const& in,
std::string const& check,
unit_test::suite& suite) const
{
auto const f =
[&](std::size_t i, std::size_t j)
{
int result;
std::string out(check.size(), 0);
::z_stream zs;
memset(&zs, 0, sizeof(zs));
result = inflateInit2(&zs, -window);
if(result != Z_OK)
{
suite.fail("! Z_OK", __FILE__, __LINE__);
return;
}
zs.next_in = (Bytef*)in.data();
zs.next_out = (Bytef*)out.data();
zs.avail_in = static_cast<uInt>(i);
zs.avail_out = static_cast<uInt>(j);
bool bi = ! (i < in.size());
bool bo = ! (j < check.size());
for(;;)
{
result = inflate(&zs, flush_);
if( result == Z_BUF_ERROR ||
result == Z_STREAM_END) // per zlib FAQ
{
out.resize(zs.total_out);
suite.expect(out == check, __FILE__, __LINE__);
break;
}
if(result != Z_OK)
{
suite.fail("! Z_OK", __FILE__, __LINE__);
break;
}
if(zs.avail_in == 0 && ! bi)
{
bi = true;
zs.avail_in = static_cast<uInt>(in.size() - i);
}
if(zs.avail_out == 0 && ! bo)
{
bo = true;
zs.avail_out = static_cast<uInt>(check.size() - j);
}
}
inflateEnd(&zs);
};
std::size_t i0, i1;
std::size_t j0, j1;
switch(in_)
{
default:
case once: i0 = in.size(); i1 = i0; break;
case half: i0 = in.size() / 2; i1 = i0; break;
case full: i0 = 1; i1 = in.size(); break;
}
switch(check_)
{
default:
case once: j0 = check.size(); j1 = j0; break;
case half: j0 = check.size() / 2; j1 = j0; break;
case full: j0 = 1; j1 = check.size(); break;
}
for(std::size_t i = i0; i <= i1; ++i)
for(std::size_t j = j0; j <= j1; ++j)
f(i, j);
}
};
class Matrix
{
unit_test::suite& suite_;
@@ -277,14 +261,9 @@ public:
template<class F>
void
operator()(
std::string label,
F const& f,
std::string const& check) const
{
using namespace std::chrono;
using clock_type = steady_clock;
auto const when = clock_type::now();
for(auto level = level_[0];
level <= level_[1]; ++level)
{
@@ -293,23 +272,14 @@ public:
{
for(auto strategy = strategy_[0];
strategy <= strategy_[1]; ++strategy)
{
z_deflator zd;
zd.level(level);
zd.windowBits(window);
zd.strategy(strategy);
auto const in = zd(check);
f(window, in, check, suite_);
f(
window,
compress(check, level, window, 4, strategy),
check,
suite_);
}
}
}
auto const elapsed = clock_type::now() - when;
suite_.log <<
label << ": " <<
duration_cast<
milliseconds>(elapsed).count() << "ms\n";
suite_.log.flush();
}
};
void
@@ -326,67 +296,57 @@ public:
" \"remoteCloseCode\": 1000,\n"
" \"reportfile\": \"autobahnpython_0_6_0_case_1_1_1.json\"\n"
;
m("1. beast", Beast{half, half}, check);
m("1. zlib ", ZLib {half, half}, check);
m(Beast{half, half}, check);
}
#if ! BOOST_BEAST_NO_SLOW_TESTS
{
Matrix m{*this};
auto const check = corpus1(50000);
m("2. beast", Beast{half, half}, check);
m("2. zlib ", ZLib {half, half}, check);
auto const check = corpus1(5000);
m(Beast{half, half}, check);
}
{
Matrix m{*this};
auto const check = corpus2(50000);
m("3. beast", Beast{half, half}, check);
m("3. zlib ", ZLib {half, half}, check);
auto const check = corpus2(5000);
m(Beast{half, half}, check);
}
{
Matrix m{*this};
auto const check = corpus1(10000);
auto const check = corpus1(1000);
m.level(6);
m.window(9);
m.strategy(Z_DEFAULT_STRATEGY);
m("4. beast", Beast{once, full}, check);
m("4. zlib ", ZLib {once, full}, check);
m(Beast{once, full}, check);
}
{
Matrix m{*this};
auto const check = corpus2(10000);
auto const check = corpus2(1000);
m.level(6);
m.window(9);
m.strategy(Z_DEFAULT_STRATEGY);
m("5. beast", Beast{once, full}, check);
m("5. zlib ", ZLib {once, full}, check);
m(Beast{once, full}, check);
}
{
Matrix m{*this};
m.level(6);
m.window(9);
auto const check = corpus1(200);
m("6. beast", Beast{full, full}, check);
m("6. zlib ", ZLib {full, full}, check);
m(Beast{full, full}, check);
}
{
Matrix m{*this};
m.level(6);
m.window(9);
auto const check = corpus2(500);
m("7. beast", Beast{full, full}, check);
m("7. zlib ", ZLib {full, full}, check);
m(Beast{full, full}, check);
}
{
Matrix m{*this};
auto const check = corpus2(10000);
auto const check = corpus2(1000);
m.level(6);
m.window(9);
m.strategy(Z_DEFAULT_STRATEGY);
m("8. beast", Beast{full, once, Flush::block}, check);
m("8. zlib ", ZLib {full, once, Z_BLOCK}, check);
m(Beast{full, once, Flush::block}, check);
}
#endif
// VFALCO Fails, but I'm unsure of what the correct
// behavior of Z_TREES/Flush::trees is.
@@ -397,8 +357,7 @@ public:
m.level(6);
m.window(9);
m.strategy(Z_DEFAULT_STRATEGY);
m("9. beast", Beast{full, once, Flush::trees}, check);
m("9. zlib ", ZLib {full, once, Z_TREES}, check);
m(Beast{full, once, Flush::trees}, check);
}
#endif
}

86
test/beast/zlib/ztest.hpp
View File

@@ -12,8 +12,6 @@
#include "zlib-1.2.11/zlib.h"
#include <cstdint>
#include <random>
#include <string>
class z_deflator
@@ -83,88 +81,4 @@ public:
}
};
class z_inflator
{
public:
std::string
operator()(std::string const& in)
{
int result;
std::string out;
z_stream zs;
memset(&zs, 0, sizeof(zs));
result = inflateInit2(&zs, -15);
try
{
zs.next_in = (Bytef*)in.data();
zs.avail_in = static_cast<uInt>(in.size());
for(;;)
{
out.resize(zs.total_out + 1024);
zs.next_out = (Bytef*)&out[zs.total_out];
zs.avail_out = static_cast<uInt>(
out.size() - zs.total_out);
result = inflate(&zs, Z_SYNC_FLUSH);
if( result == Z_NEED_DICT ||
result == Z_DATA_ERROR ||
result == Z_MEM_ERROR)
{
throw std::logic_error("inflate failed");
}
if(zs.avail_out > 0)
break;
if(result == Z_STREAM_END)
break;
}
out.resize(zs.total_out);
inflateEnd(&zs);
}
catch(...)
{
inflateEnd(&zs);
throw;
}
return out;
}
};
// Lots of repeats, limited char range
inline
std::string
corpus1(std::size_t n)
{
static std::string const alphabet{
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
};
std::string s;
s.reserve(n + 5);
std::mt19937 g;
std::uniform_int_distribution<std::size_t> d0{
0, alphabet.size() - 1};
std::uniform_int_distribution<std::size_t> d1{
1, 5};
while(s.size() < n)
{
auto const rep = d1(g);
auto const ch = alphabet[d0(g)];
s.insert(s.end(), rep, ch);
}
s.resize(n);
return s;
}
// Random data
inline
std::string
corpus2(std::size_t n)
{
std::string s;
s.reserve(n);
std::mt19937 g;
std::uniform_int_distribution<std::uint32_t> d0{0, 255};
while(n--)
s.push_back(static_cast<char>(d0(g)));
return s;
}
#endif