Faster zlib tests

CHANGELOG.md

@@ -7,6 +7,7 @@ Version 96:
 * Fix library.json
 * Update boostorg links
 * Add bench-zlib
+* Faster zlib tests
 
 --------------------------------------------------------------------------------
 

test/beast/zlib/deflate_stream.cpp

@@ -10,9 +10,14 @@
 // Test that header file is self-contained.
 #include <boost/beast/zlib/deflate_stream.hpp>
 
-#include "ztest.hpp"
+#include <boost/beast/core/string.hpp>
-
 #include <boost/beast/unit_test/suite.hpp>
+#include <cstdint>
+#include <random>
+
+#include "zlib-1.2.11/zlib.h"
+
+#include "ztest.hpp"
 
 namespace boost {
 namespace beast {
@@ -21,6 +26,127 @@ namespace zlib {
 class deflate_stream_test : public beast::unit_test::suite
 {
 public:
+    // Lots of repeats, limited char range
+    static
+    std::string
+    corpus1(std::size_t n)
+    {
+        static std::string const alphabet{
+            "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+        };
+        std::string s;
+        s.reserve(n + 5);
+        std::mt19937 g;
+        std::uniform_int_distribution<std::size_t> d0{
+            0, alphabet.size() - 1};
+        std::uniform_int_distribution<std::size_t> d1{
+            1, 5};
+        while(s.size() < n)
+        {
+            auto const rep = d1(g);
+            auto const ch = alphabet[d0(g)];
+            s.insert(s.end(), rep, ch);
+        }
+        s.resize(n);
+        return s;
+    }
+
+    // Random data
+    static
+    std::string
+    corpus2(std::size_t n)
+    {
+        std::string s;
+        s.reserve(n);
+        std::mt19937 g;
+        std::uniform_int_distribution<std::uint32_t> d0{0, 255};
+        while(n--)
+            s.push_back(static_cast<char>(d0(g)));
+        return s;
+    }
+
+    static
+    std::string
+    compress(
+        string_view const& in,
+        int level,                  // 0=none, 1..9, -1=default
+        int windowBits,             // 9..15
+        int memLevel)               // 1..9 (8=default)
+    {
+        int const strategy = Z_DEFAULT_STRATEGY;
+        int result;
+        z_stream zs;
+        memset(&zs, 0, sizeof(zs));
+        result = deflateInit2(
+            &zs,
+            level,
+            Z_DEFLATED,
+            -windowBits,
+            memLevel,
+            strategy);
+        if(result != Z_OK)
+            throw std::logic_error{"deflateInit2 failed"};
+        zs.next_in = (Bytef*)in.data();
+        zs.avail_in = static_cast<uInt>(in.size());
+        std::string out;
+        out.resize(deflateBound(&zs,
+            static_cast<uLong>(in.size())));
+        zs.next_in = (Bytef*)in.data();
+        zs.avail_in = static_cast<uInt>(in.size());
+        zs.next_out = (Bytef*)&out[0];
+        zs.avail_out = static_cast<uInt>(out.size());
+        result = deflate(&zs, Z_FULL_FLUSH);
+        if(result != Z_OK)
+            throw std::logic_error("deflate failed");
+        out.resize(zs.total_out);
+        deflateEnd(&zs);
+        return out;
+    }
+
+    static
+    std::string
+    decompress(string_view const& in)
+    {
+        int result;
+        std::string out;
+        z_stream zs;
+        memset(&zs, 0, sizeof(zs));
+        result = inflateInit2(&zs, -15);
+        try
+        {
+            zs.next_in = (Bytef*)in.data();
+            zs.avail_in = static_cast<uInt>(in.size());
+            for(;;)
+            {
+                out.resize(zs.total_out + 1024);
+                zs.next_out = (Bytef*)&out[zs.total_out];
+                zs.avail_out = static_cast<uInt>(
+                    out.size() - zs.total_out);
+                result = inflate(&zs, Z_SYNC_FLUSH);
+                if( result == Z_NEED_DICT ||
+                    result == Z_DATA_ERROR ||
+                    result == Z_MEM_ERROR)
+                {
+                    throw std::logic_error("inflate failed");
+                }
+                if(zs.avail_out > 0)
+                    break;
+                if(result == Z_STREAM_END)
+                    break;
+            }
+            out.resize(zs.total_out);
+            inflateEnd(&zs);
+        }
+        catch(...)
+        {
+            inflateEnd(&zs);
+            throw;
+        }
+        return out;
+    }
+
+    //--------------------------------------------------------------------------
+
     using self = deflate_stream_test;
     typedef void(self::*pmf_t)(
         int level, int windowBits, int strategy,
@@ -41,55 +167,6 @@ public:
         }
     }
 
-    void
-    doDeflate1_zlib(
-        int level, int windowBits, int strategy,
-            std::string const& check)
-    {
-        int result;
-        std::string out;
-        ::z_stream zs;
-        std::memset(&zs, 0, sizeof(zs));
-        result = deflateInit2(&zs,
-            level,
-            Z_DEFLATED,
-            -windowBits,
-            8,
-            strategy);
-        if(! BEAST_EXPECT(result == Z_OK))
-            goto err;
-        out.resize(deflateBound(&zs,
-            static_cast<uLong>(check.size())));
-        zs.next_in = (Bytef*)check.data();
-        zs.avail_in = static_cast<uInt>(check.size());
-        zs.next_out = (Bytef*)out.data();
-        zs.avail_out = static_cast<uInt>(out.size());
-        {
-            bool progress = true;
-            for(;;)
-            {
-                result = deflate(&zs, Z_FULL_FLUSH);
-                if( result == Z_BUF_ERROR ||
-                    result == Z_STREAM_END) // per zlib FAQ
-                    goto fin;
-                if(! BEAST_EXPECT(progress))
-                    goto err;
-                progress = false;
-            }
-        }
-
-    fin:
-        out.resize(zs.total_out);
-        {
-            z_inflator zi;
-            auto const s = zi(out);
-            BEAST_EXPECT(s == check);
-        }
-
-    err:
-        deflateEnd(&zs);
-    }
-
     void
     doDeflate1_beast(
         int level, int windowBits, int strategy,
@@ -128,11 +205,7 @@ public:
 
     fin:
         out.resize(zs.total_out);
-        {
+        BEAST_EXPECT(decompress(out) == check);
-            z_inflator zi;
-            auto const s = zi(out);
-            BEAST_EXPECT(s == check);
-        }
 
     err:
         ;
@@ -140,77 +213,6 @@ public:
 
     //--------------------------------------------------------------------------
 
-    void
-    doDeflate2_zlib(
-        int level, int windowBits, int strategy,
-            std::string const& check)
-    {
-        for(std::size_t i = 1; i < check.size(); ++i)
-        {
-            for(std::size_t j = 1;; ++j)
-            {
-                int result;
-                ::z_stream zs;
-                std::memset(&zs, 0, sizeof(zs));
-                result = deflateInit2(&zs,
-                    level,
-                    Z_DEFLATED,
-                    -windowBits,
-                    8,
-                    strategy);
-                if(! BEAST_EXPECT(result == Z_OK))
-                    continue;
-                std::string out;
-                out.resize(deflateBound(&zs,
-                    static_cast<uLong>(check.size())));
-                if(j >= out.size())
-                {
-                    deflateEnd(&zs);
-                    break;
-                }
-                zs.next_in = (Bytef*)check.data();
-                zs.avail_in = static_cast<uInt>(i);
-                zs.next_out = (Bytef*)out.data();
-                zs.avail_out = static_cast<uInt>(j);
-                bool bi = false;
-                bool bo = false;
-                for(;;)
-                {
-                    int flush = bi ? Z_FULL_FLUSH : Z_NO_FLUSH;
-                    result = deflate(&zs, flush);
-                    if( result == Z_BUF_ERROR ||
-                        result == Z_STREAM_END) // per zlib FAQ
-                        goto fin;
-                    if(! BEAST_EXPECT(result == Z_OK))
-                        goto err;
-                    if(zs.avail_in == 0 && ! bi)
-                    {
-                        bi = true;
-                        zs.avail_in =
-                            static_cast<uInt>(check.size() - i);
-                    }
-                    if(zs.avail_out == 0 && ! bo)
-                    {
-                        bo = true;
-                        zs.avail_out =
-                            static_cast<uInt>(out.size() - j);
-                    }
-                }
-
-            fin:
-                out.resize(zs.total_out);
-                {
-                    z_inflator zi;
-                    auto const s = zi(out);
-                    BEAST_EXPECT(s == check);
-                }
-
-            err:
-                deflateEnd(&zs);
-            }
-        }
-    }
-
     void
     doDeflate2_beast(
         int level, int windowBits, int strategy,
@@ -264,11 +266,7 @@ public:
 
             fin:
                 out.resize(zs.total_out);
-                {
+                BEAST_EXPECT(decompress(out) == check);
-                    z_inflator zi;
-                    auto const s = zi(out);
-                    BEAST_EXPECT(s == check);
-                }
 
             err:
                 ;
@@ -279,12 +277,8 @@ public:
     //--------------------------------------------------------------------------
 
     void
-    doMatrix(std::string const& label,
+    doMatrix(std::string const& check, pmf_t pmf)
-        std::string const& check, pmf_t pmf)
     {
-        using namespace std::chrono;
-        using clock_type = steady_clock;
-        auto const when = clock_type::now();
         for(int level = 0; level <= 9; ++level)
         {
             for(int windowBits = 8; windowBits <= 9; ++windowBits)
@@ -299,33 +293,15 @@ public:
                 }
             }
         }
-        auto const elapsed = clock_type::now() - when;
-        log <<
-            label << ": " <<
-            duration_cast<
-                milliseconds>(elapsed).count() << "ms\n";
-        log.flush();
     }
 
     void
     testDeflate()
     {
-        doMatrix("1.beast ", "Hello, world!", &self::doDeflate1_beast);
+        doMatrix("Hello, world!", &self::doDeflate1_beast);
-        doMatrix("1.zlib  ", "Hello, world!", &self::doDeflate1_zlib);
+        doMatrix("Hello, world!", &self::doDeflate2_beast);
-    #if ! BOOST_BEAST_NO_SLOW_TESTS
+        doMatrix(corpus1(56), &self::doDeflate2_beast);
-        doMatrix("2.beast ", "Hello, world!", &self::doDeflate2_beast);
+        doMatrix(corpus1(1024), &self::doDeflate1_beast);
-        doMatrix("2.zlib  ", "Hello, world!", &self::doDeflate2_zlib);
-        {
-            auto const s = corpus1(56);
-            doMatrix("3.beast ", s, &self::doDeflate2_beast);
-            doMatrix("3.zlib  ", s, &self::doDeflate2_zlib);
-        }
-        {
-            auto const s = corpus1(512 * 1024);
-            doMatrix("4.beast ", s, &self::doDeflate1_beast);
-            doMatrix("4.zlib  ", s, &self::doDeflate1_zlib);
-        }
-    #endif
     }
 
     void

test/beast/zlib/inflate_stream.cpp

@@ -10,12 +10,13 @@
 // Test that header file is self-contained.
 #include <boost/beast/zlib/inflate_stream.hpp>
 
-#include "ztest.hpp"
+#include <boost/beast/core/string.hpp>
-
 #include <boost/beast/unit_test/suite.hpp>
 #include <chrono>
 #include <random>
 
+#include "ztest.hpp"
+
 namespace boost {
 namespace beast {
 namespace zlib {
@@ -23,6 +24,83 @@ namespace zlib {
 class inflate_stream_test : public beast::unit_test::suite
 {
 public:
+    // Lots of repeats, limited char range
+    static
+    std::string
+    corpus1(std::size_t n)
+    {
+        static std::string const alphabet{
+            "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+        };
+        std::string s;
+        s.reserve(n + 5);
+        std::mt19937 g;
+        std::uniform_int_distribution<std::size_t> d0{
+            0, alphabet.size() - 1};
+        std::uniform_int_distribution<std::size_t> d1{
+            1, 5};
+        while(s.size() < n)
+        {
+            auto const rep = d1(g);
+            auto const ch = alphabet[d0(g)];
+            s.insert(s.end(), rep, ch);
+        }
+        s.resize(n);
+        return s;
+    }
+
+    // Random data
+    static
+    std::string
+    corpus2(std::size_t n)
+    {
+        std::string s;
+        s.reserve(n);
+        std::mt19937 g;
+        std::uniform_int_distribution<std::uint32_t> d0{0, 255};
+        while(n--)
+            s.push_back(static_cast<char>(d0(g)));
+        return s;
+    }
+
+    static
+    std::string
+    compress(
+        string_view const& in,
+        int level,                  // 0=none, 1..9, -1=default
+        int windowBits,             // 9..15
+        int memLevel,               // 1..9 (8=default)
+        int strategy)               // e.g. Z_DEFAULT_STRATEGY
+    {
+        int result;
+        z_stream zs;
+        memset(&zs, 0, sizeof(zs));
+        result = deflateInit2(
+            &zs,
+            level,
+            Z_DEFLATED,
+            -windowBits,
+            memLevel,
+            strategy);
+        if(result != Z_OK)
+            throw std::logic_error{"deflateInit2 failed"};
+        zs.next_in = (Bytef*)in.data();
+        zs.avail_in = static_cast<uInt>(in.size());
+        std::string out;
+        out.resize(deflateBound(&zs,
+            static_cast<uLong>(in.size())));
+        zs.next_in = (Bytef*)in.data();
+        zs.avail_in = static_cast<uInt>(in.size());
+        zs.next_out = (Bytef*)&out[0];
+        zs.avail_out = static_cast<uInt>(out.size());
+        result = deflate(&zs, Z_FULL_FLUSH);
+        if(result != Z_OK)
+            throw std::logic_error("deflate failed");
+        out.resize(zs.total_out);
+        deflateEnd(&zs);
+        return out;
+    }
+
     //--------------------------------------------------------------------------
 
     enum Split
@@ -120,100 +198,6 @@ public:
         }
     };
 
-    class ZLib
-    {
-        Split in_;
-        Split check_;
-        int flush_;
-
-    public:
-        ZLib(Split in, Split check, int flush = Z_SYNC_FLUSH)
-            : in_(in)
-            , check_(check)
-            , flush_(flush)
-        {
-        }
-
-        void
-        operator()(
-            int window,
-            std::string const& in,
-            std::string const& check,
-            unit_test::suite& suite) const
-        {
-            auto const f =
-            [&](std::size_t i, std::size_t j)
-            {
-                int result;
-                std::string out(check.size(), 0);
-                ::z_stream zs;
-                memset(&zs, 0, sizeof(zs));
-                result = inflateInit2(&zs, -window);
-                if(result != Z_OK)
-                {
-                    suite.fail("! Z_OK", __FILE__, __LINE__);
-                    return;
-                }
-                zs.next_in = (Bytef*)in.data();
-                zs.next_out = (Bytef*)out.data();
-                zs.avail_in = static_cast<uInt>(i);
-                zs.avail_out = static_cast<uInt>(j);
-                bool bi = ! (i < in.size());
-                bool bo = ! (j < check.size());
-                for(;;)
-                {
-                    result = inflate(&zs, flush_);
-                    if( result == Z_BUF_ERROR ||
-                        result == Z_STREAM_END) // per zlib FAQ
-                    {
-                        out.resize(zs.total_out);
-                        suite.expect(out == check, __FILE__, __LINE__);
-                        break;
-                    }
-                    if(result != Z_OK)
-                    {
-                        suite.fail("! Z_OK", __FILE__, __LINE__);
-                        break;
-                    }
-                    if(zs.avail_in == 0 && ! bi)
-                    {
-                        bi = true;
-                        zs.avail_in = static_cast<uInt>(in.size() - i);
-                    }
-                    if(zs.avail_out == 0 && ! bo)
-                    {
-                        bo = true;
-                        zs.avail_out = static_cast<uInt>(check.size() - j);
-                    }
-                }
-                inflateEnd(&zs);
-            };
-
-            std::size_t i0, i1;
-            std::size_t j0, j1;
-
-            switch(in_)
-            {
-            default:
-            case once: i0 = in.size();     i1 = i0;         break;
-            case half: i0 = in.size() / 2; i1 = i0;         break;
-            case full: i0 = 1;             i1 = in.size();  break;
-            }
-
-            switch(check_)
-            {
-            default:
-            case once: j0 = check.size();     j1 = j0;              break;
-            case half: j0 = check.size() / 2; j1 = j0;              break;
-            case full: j0 = 1;                j1 = check.size();    break;
-            }
-
-            for(std::size_t i = i0; i <= i1; ++i)
-                for(std::size_t j = j0; j <= j1; ++j)
-                    f(i, j);
-        }
-    };
-
     class Matrix
     {
         unit_test::suite& suite_;
@@ -277,14 +261,9 @@ public:
         template<class F>
         void
         operator()(
-            std::string label,
             F const& f,
             std::string const& check) const
         {
-            using namespace std::chrono;
-            using clock_type = steady_clock;
-            auto const when = clock_type::now();
-
             for(auto level = level_[0];
                 level <= level_[1]; ++level)
             {
@@ -293,23 +272,14 @@ public:
                 {
                     for(auto strategy = strategy_[0];
                         strategy <= strategy_[1]; ++strategy)
-                    {
+                        f(
-                        z_deflator zd;
+                            window,
-                        zd.level(level);
+                            compress(check, level, window, 4, strategy),
-                        zd.windowBits(window);
+                            check,
-                        zd.strategy(strategy);
+                            suite_);
-                        auto const in = zd(check);
-                        f(window, in, check, suite_);
                 }
             }
         }
-            auto const elapsed = clock_type::now() - when;
-            suite_.log <<
-                label << ": " <<
-                duration_cast<
-                    milliseconds>(elapsed).count() << "ms\n";
-            suite_.log.flush();
-        }
     };
 
     void
@@ -326,67 +296,57 @@ public:
                 "         \"remoteCloseCode\": 1000,\n"
                 "         \"reportfile\": \"autobahnpython_0_6_0_case_1_1_1.json\"\n"
                 ;
-            m("1. beast", Beast{half, half}, check);
+            m(Beast{half, half}, check);
-            m("1. zlib ", ZLib {half, half}, check);
         }
 
-    #if ! BOOST_BEAST_NO_SLOW_TESTS
         {
             Matrix m{*this};
-            auto const check = corpus1(50000);
+            auto const check = corpus1(5000);
-            m("2. beast", Beast{half, half}, check);
+            m(Beast{half, half}, check);
-            m("2. zlib ", ZLib {half, half}, check);
         }
         {
             Matrix m{*this};
-            auto const check = corpus2(50000);
+            auto const check = corpus2(5000);
-            m("3. beast", Beast{half, half}, check);
+            m(Beast{half, half}, check);
-            m("3. zlib ", ZLib {half, half}, check);
         }
         {
             Matrix m{*this};
-            auto const check = corpus1(10000);
+            auto const check = corpus1(1000);
             m.level(6);
             m.window(9);
             m.strategy(Z_DEFAULT_STRATEGY);
-            m("4. beast", Beast{once, full}, check);
+            m(Beast{once, full}, check);
-            m("4. zlib ", ZLib {once, full}, check);
         }
         {
             Matrix m{*this};
-            auto const check = corpus2(10000);
+            auto const check = corpus2(1000);
             m.level(6);
             m.window(9);
             m.strategy(Z_DEFAULT_STRATEGY);
-            m("5. beast", Beast{once, full}, check);
+            m(Beast{once, full}, check);
-            m("5. zlib ", ZLib {once, full}, check);
         }
         {
             Matrix m{*this};
             m.level(6);
             m.window(9);
             auto const check = corpus1(200);
-            m("6. beast", Beast{full, full}, check);
+            m(Beast{full, full}, check);
-            m("6. zlib ", ZLib {full, full}, check);
         }
         {
             Matrix m{*this};
             m.level(6);
             m.window(9);
             auto const check = corpus2(500);
-            m("7. beast", Beast{full, full}, check);
+            m(Beast{full, full}, check);
-            m("7. zlib ", ZLib {full, full}, check);
         }
         {
             Matrix m{*this};
-            auto const check = corpus2(10000);
+            auto const check = corpus2(1000);
             m.level(6);
             m.window(9);
             m.strategy(Z_DEFAULT_STRATEGY);
-            m("8. beast", Beast{full, once, Flush::block}, check);
+            m(Beast{full, once, Flush::block}, check);
-            m("8. zlib ", ZLib {full, once, Z_BLOCK}, check);
         }
-    #endif
 
         // VFALCO Fails, but I'm unsure of what the correct
         //        behavior of Z_TREES/Flush::trees is.
@@ -397,8 +357,7 @@ public:
             m.level(6);
             m.window(9);
             m.strategy(Z_DEFAULT_STRATEGY);
-            m("9. beast", Beast{full, once, Flush::trees}, check);
+            m(Beast{full, once, Flush::trees}, check);
-            m("9. zlib ", ZLib {full, once, Z_TREES}, check);
         }
 #endif
     }

test/beast/zlib/ztest.hpp

@@ -12,8 +12,6 @@
 
 #include "zlib-1.2.11/zlib.h"
 
-#include <cstdint>
-#include <random>
 #include <string>
 
 class z_deflator
@@ -83,88 +81,4 @@ public:
     }
 };
 
-class z_inflator
-{
-public:
-    std::string
-    operator()(std::string const& in)
-    {
-        int result;
-        std::string out;
-        z_stream zs;
-        memset(&zs, 0, sizeof(zs));
-        result = inflateInit2(&zs, -15);
-        try
-        {
-            zs.next_in = (Bytef*)in.data();
-            zs.avail_in = static_cast<uInt>(in.size());
-            for(;;)
-            {
-                out.resize(zs.total_out + 1024);
-                zs.next_out = (Bytef*)&out[zs.total_out];
-                zs.avail_out = static_cast<uInt>(
-                    out.size() - zs.total_out);
-                result = inflate(&zs, Z_SYNC_FLUSH);
-                if( result == Z_NEED_DICT ||
-                    result == Z_DATA_ERROR ||
-                    result == Z_MEM_ERROR)
-                {
-                    throw std::logic_error("inflate failed");
-                }
-                if(zs.avail_out > 0)
-                    break;
-                if(result == Z_STREAM_END)
-                    break;
-            }
-            out.resize(zs.total_out);
-            inflateEnd(&zs);
-        }
-        catch(...)
-        {
-            inflateEnd(&zs);
-            throw;
-        }
-        return out;
-    }
-};
-
-// Lots of repeats, limited char range
-inline
-std::string
-corpus1(std::size_t n)
-{
-    static std::string const alphabet{
-        "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
-    };
-    std::string s;
-    s.reserve(n + 5);
-    std::mt19937 g;
-    std::uniform_int_distribution<std::size_t> d0{
-        0, alphabet.size() - 1};
-    std::uniform_int_distribution<std::size_t> d1{
-        1, 5};
-    while(s.size() < n)
-    {
-        auto const rep = d1(g);
-        auto const ch = alphabet[d0(g)];
-        s.insert(s.end(), rep, ch);
-    }
-    s.resize(n);
-    return s;
-}
-
-// Random data
-inline
-std::string
-corpus2(std::size_t n)
-{
-    std::string s;
-    s.reserve(n);
-    std::mt19937 g;
-    std::uniform_int_distribution<std::uint32_t> d0{0, 255};
-    while(n--)
-        s.push_back(static_cast<char>(d0(g)));
-    return s;
-}
-
 #endif