From f00a29d3df4a9c4e34eb856e4a0c14d5d9a04896 Mon Sep 17 00:00:00 2001 From: Christian Mazakas Date: Tue, 21 Jun 2022 15:33:07 -0700 Subject: [PATCH 1/6] Add tests for the internal prime_fmod_size policy --- test/Jamfile.v2 | 1 + test/unordered/prime_fmod_tests.cpp | 247 ++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 test/unordered/prime_fmod_tests.cpp diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 1bf3d858..50c1c8cd 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -29,6 +29,7 @@ project msvc:on ; +run unordered/prime_fmod_tests.cpp ; run unordered/fwd_set_test.cpp ; run unordered/fwd_map_test.cpp ; run unordered/allocator_traits.cpp ; diff --git a/test/unordered/prime_fmod_tests.cpp b/test/unordered/prime_fmod_tests.cpp new file mode 100644 index 00000000..09b7ead1 --- /dev/null +++ b/test/unordered/prime_fmod_tests.cpp @@ -0,0 +1,247 @@ +// Copyright 2022 Christian Mazakas. +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#include +#include +#include + +#include + +#if defined(BOOST_MSVC) +// conditional expression is constant +#pragma warning(disable : 4127) +#endif + +void macros_test() +{ + if (std::numeric_limits::digits >= 64) { +#if !defined(BOOST_UNORDERED_FCA_HAS_64B_SIZE_T) + BOOST_ERROR("std::numeric_limits::digits >= 64, but " + "BOOST_UNORDERED_FCA_HAS_64B_SIZE_T is not defined"); +#endif + } + +#if ((defined(__GNUC__) || defined(__clang__)) && \ + defined(__SIZEOF_INT128__)) || \ + (defined(_MSC_VER) && defined(_M_X64)) +#if !defined(BOOST_UNORDERED_FCA_FASTMOD_SUPPORT) + BOOST_ERROR("fast_modulo support should be enabled for this toolchain but " + "it's currently not"); +#endif +#endif +} + +// Pretty inefficient, but the test is fast enough. +// Might be too slow if we had larger primes? +bool is_prime(std::size_t x) +{ + if (x == 2) { + return true; + } + + if (x == 1 || x % 2 == 0) { + return false; + } + + // y*y <= x is susceptible to overflow, so instead make sure to use y <= (x/y) + for (std::size_t y = 3; y <= (x / y); y += 2) { + if (x % y == 0) { + return false; + } + } + + return true; +} + +void prime_sizes_test() +{ + // just some basic sanity checks + // + BOOST_TEST(!is_prime(0)); + BOOST_TEST(!is_prime(1)); + BOOST_TEST(is_prime(2)); + BOOST_TEST(is_prime(3)); + BOOST_TEST(is_prime(13)); + BOOST_TEST(!is_prime(4)); + BOOST_TEST(!is_prime(100)); + BOOST_TEST(!is_prime(49)); + + std::size_t* sizes = boost::unordered::detail::prime_fmod_size<>::sizes; + std::size_t sizes_len = + boost::unordered::detail::prime_fmod_size<>::sizes_len; + + // prove every number in our sizes array is prime + // + BOOST_TEST_GT(sizes_len, 0u); + + for (std::size_t i = 0; i < sizes_len; ++i) { + BOOST_TEST(is_prime(sizes[i])); + } + + // prove that every subsequent number in the sequence is larger than the + // previous + // + for (std::size_t i = 1; i < sizes_len; ++i) { + BOOST_TEST_GT(sizes[i], sizes[i - 1]); + } + +#if defined(BOOST_UNORDERED_FCA_FASTMOD_SUPPORT) + // now we wish to prove that if we do have the reciprocals stored, we have the + // correct amount of them, i.e. one for every entry in sizes[] that fits in 32 + // bits + // + boost::uint64_t* inv_sizes32 = + boost::unordered::detail::prime_fmod_size<>::inv_sizes32; + + std::size_t inv_sizes32_len = + boost::unordered::detail::prime_fmod_size<>::inv_sizes32_len; + + std::size_t count = 0; + for (std::size_t i = 0; i < sizes_len; ++i) { + if (sizes[i] <= UINT32_MAX) { + ++count; + } + } + + BOOST_TEST_GT(inv_sizes32_len, 0u); + BOOST_TEST_EQ(inv_sizes32_len, count); + + // these values should also be monotonically decreasing + // + for (std::size_t i = 1; i < inv_sizes32_len; ++i) { + BOOST_TEST_LT(inv_sizes32[i], inv_sizes32[i - 1]); + } + + // now make sure the values in inv_sizes32 are what they should be as derived + // from the paper + // + for (std::size_t i = 0; i < inv_sizes32_len; ++i) { + std::size_t const size = sizes[i]; + BOOST_TEST_LE(size, UINT_MAX); + + boost::uint32_t d = static_cast(sizes[i]); + boost::uint64_t M = ((boost::ulong_long_type(0xffffffff) << 32) + + boost::ulong_long_type(0xffffffff)) / + d + + 1; + + BOOST_TEST_EQ(inv_sizes32[i], M); + } +#endif +} + +void get_remainder_test() +{ +#if defined(BOOST_UNORDERED_FCA_FASTMOD_SUPPORT) + struct + { + // internally, our get_remainder() function will rely on either msvc + // intrinsics or 128 bit integer support which isn't always available. This + // is a slower, portable version we can use for verification of the + // routines. + // + boost::uint64_t operator()(boost::uint64_t f, boost::uint32_t d) + { + boost::uint64_t r1 = (f & UINT32_MAX) * d; + boost::uint64_t r2 = (f >> 32) * d; + + r2 += r1 >> 32; + + return r2 >> 32; + } + } get_remainder; + + boost::detail::splitmix64 rng; + + for (std::size_t i = 0; i < 1000000u; ++i) { + boost::uint64_t f = rng(); + boost::uint32_t d = static_cast(rng()); + + boost::uint64_t r1 = + boost::unordered::detail::prime_fmod_size<>::get_remainder(f, d); + + boost::uint64_t r2 = get_remainder(f, d); + + if (!BOOST_TEST_EQ(r1, r2)) { + std::cerr << "f: " << f << ", d: " << d << std::endl; + return; + } + } +#endif +} + +void modulo32_test() +{ + std::size_t* sizes = boost::unordered::detail::prime_fmod_size<>::sizes; + + std::size_t const sizes_len = + boost::unordered::detail::prime_fmod_size<>::sizes_len; + + boost::detail::splitmix64 rng; + + for (std::size_t i = 0; i < 1000000u; ++i) { + std::size_t hash = static_cast(rng()); + + for (std::size_t j = 0; j < sizes_len; ++j) { + std::size_t p1 = + boost::unordered::detail::prime_fmod_size<>::position(hash, j); + + std::size_t p2 = hash % sizes[j]; + if (!BOOST_TEST_EQ(p1, p2)) { + std::cerr << "hash: " << hash << ", j: " << j << ", sizes[" << j + << "]: " << sizes[j] << std::endl; + return; + } + } + } +} + +void modulo64_test() +{ +#if defined(BOOST_UNORDERED_FCA_HAS_64B_SIZE_T) + std::size_t* sizes = boost::unordered::detail::prime_fmod_size<>::sizes; + + std::size_t const sizes_len = + boost::unordered::detail::prime_fmod_size<>::sizes_len; + + boost::detail::splitmix64 rng; + + for (std::size_t i = 0; i < 1000000u; ++i) { + std::size_t hash = rng(); + + for (std::size_t j = 0; j < sizes_len; ++j) { + std::size_t h = hash; + +#if defined(BOOST_UNORDERED_FCA_FASTMOD_SUPPORT) + if (sizes[j] <= UINT_MAX) { + h = boost::uint32_t(h) + boost::uint32_t(h >> 32); + } +#endif + std::size_t p1 = + boost::unordered::detail::prime_fmod_size<>::position(hash, j); + + std::size_t p2 = h % sizes[j]; + + if (!BOOST_TEST_EQ(p1, p2)) { + std::cerr << "hash: " << hash << ", j: " << j << ", h: " << h + << ", sizes[" << j << "]: " << sizes[j] << std::endl; + return; + } + } + } +#endif +} + +int main() +{ + macros_test(); + prime_sizes_test(); + modulo32_test(); + get_remainder_test(); + modulo64_test(); + + return boost::report_errors(); +} From 0f71fe28a22c13e303504d289bedf38f65af03c0 Mon Sep 17 00:00:00 2001 From: Peter Dimov Date: Sat, 25 Jun 2022 01:04:22 +0300 Subject: [PATCH 2/6] Fix typos; do not undefine macros needed for tests --- include/boost/unordered/detail/fca.hpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/include/boost/unordered/detail/fca.hpp b/include/boost/unordered/detail/fca.hpp index a255f29b..6b398070 100644 --- a/include/boost/unordered/detail/fca.hpp +++ b/include/boost/unordered/detail/fca.hpp @@ -378,16 +378,8 @@ namespace boost { #undef BOOST_UNORDERED_PRIME_FMOD_POSITIONS_ELEMENT #undef BOOST_UNORDERED_PRIME_FMOD_SIZES #undef BOOST_UNORDERED_PRIME_FMOD_SIZES_64BIT -#undef BOOST_UNORDERED_PRIME_FMOD_SIZES_34BIT -#undef BOOST_UNORDERED_PRIME_FMOD_SIZES_34BIT_INCOMPLETE - -#ifdef BOOST_UNORDERED_FCA_FASTMOD_SUPPORT -#undef BOOST_UNORDERED_FCA_FASTMOD_SUPPORT -#endif - -#ifdef BOOST_UNORDERED_FCA_HAS_64B_SIZE_T -#undef BOOST_UNORDERED_FCA_HAS_64B_SIZE_T -#endif +#undef BOOST_UNORDERED_PRIME_FMOD_SIZES_32BIT +#undef BOOST_UNORDERED_PRIME_FMOD_SIZES_32BIT_INCOMPLETE template struct node { From 31cffd84125f1deac3eb727065fbd8c126aa0672 Mon Sep 17 00:00:00 2001 From: Peter Dimov Date: Sat, 25 Jun 2022 01:06:15 +0300 Subject: [PATCH 3/6] Fix reversed condition --- include/boost/unordered/detail/fca.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/unordered/detail/fca.hpp b/include/boost/unordered/detail/fca.hpp index 6b398070..11902eba 100644 --- a/include/boost/unordered/detail/fca.hpp +++ b/include/boost/unordered/detail/fca.hpp @@ -344,7 +344,7 @@ namespace boost { (boost::ulong_long_type(5ul) << 32) + boost::ulong_long_type(1431653234ul) /* = 22906489714 */, (boost::ulong_long_type(2ul) << 32) + boost::ulong_long_type(2863311496ul) /* = 11453246088 */, (boost::ulong_long_type(1ul) << 32) + boost::ulong_long_type(1431655764ul) /* = 5726623060 */, -#if !defined(BOOST_UNORDERED_FCA_HAS_64B_SIZE_T) +#if defined(BOOST_UNORDERED_FCA_HAS_64B_SIZE_T) }; #else (boost::ulong_long_type(1ul) << 32) + boost::ulong_long_type(6ul) /* 4294967302 */ From c53e0228c52cc4d3adfde2eaece98c5eebe42a95 Mon Sep 17 00:00:00 2001 From: Peter Dimov Date: Sat, 25 Jun 2022 01:16:32 +0300 Subject: [PATCH 4/6] Check BOOST_UNORDERED_FCA_HAS_64B_SIZE_T in the 32 bit case as well --- test/unordered/prime_fmod_tests.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unordered/prime_fmod_tests.cpp b/test/unordered/prime_fmod_tests.cpp index 09b7ead1..149bdc5a 100644 --- a/test/unordered/prime_fmod_tests.cpp +++ b/test/unordered/prime_fmod_tests.cpp @@ -21,6 +21,12 @@ void macros_test() #if !defined(BOOST_UNORDERED_FCA_HAS_64B_SIZE_T) BOOST_ERROR("std::numeric_limits::digits >= 64, but " "BOOST_UNORDERED_FCA_HAS_64B_SIZE_T is not defined"); +#endif + } + else { +#if defined(BOOST_UNORDERED_FCA_HAS_64B_SIZE_T) + BOOST_ERROR("std::numeric_limits::digits < 64, but " + "BOOST_UNORDERED_FCA_HAS_64B_SIZE_T is defined"); #endif } From d204b9b4082f6dfad6b616670c99feaf51643942 Mon Sep 17 00:00:00 2001 From: Peter Dimov Date: Sat, 25 Jun 2022 01:17:50 +0300 Subject: [PATCH 5/6] Remove unnecessary include --- test/unordered/prime_fmod_tests.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unordered/prime_fmod_tests.cpp b/test/unordered/prime_fmod_tests.cpp index 149bdc5a..231fd913 100644 --- a/test/unordered/prime_fmod_tests.cpp +++ b/test/unordered/prime_fmod_tests.cpp @@ -6,7 +6,6 @@ #include #include -#include #include From 08e0fee1418d441b25c964aed48eb6e7f6cc56f0 Mon Sep 17 00:00:00 2001 From: Peter Dimov Date: Sat, 25 Jun 2022 01:44:14 +0300 Subject: [PATCH 6/6] Enable fastmod on clang-cl and other pretenders such as Intel --- include/boost/unordered/detail/fca.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/unordered/detail/fca.hpp b/include/boost/unordered/detail/fca.hpp index 11902eba..df3e4688 100644 --- a/include/boost/unordered/detail/fca.hpp +++ b/include/boost/unordered/detail/fca.hpp @@ -151,7 +151,7 @@ namespace boost { #endif #if !defined(BOOST_NO_INT64_T) && \ - (defined(BOOST_HAS_INT128) || (defined(BOOST_MSVC) && defined(_WIN64))) + (defined(BOOST_HAS_INT128) || (defined(_MSC_VER) && defined(_M_X64))) #define BOOST_UNORDERED_FCA_FASTMOD_SUPPORT #endif