From 3a8bea9e1df16f972218e5cfdad24cf1021a7629 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Sat, 11 Mar 2023 18:36:09 +0100 Subject: [PATCH] parameterized group15 and core --- include/boost/unordered/detail/foa/core.hpp | 103 ++++++++++++------- include/boost/unordered/detail/foa/table.hpp | 33 ++++-- 2 files changed, 91 insertions(+), 45 deletions(-) diff --git a/include/boost/unordered/detail/foa/core.hpp b/include/boost/unordered/detail/foa/core.hpp index c4fb08f4..f316fab5 100644 --- a/include/boost/unordered/detail/foa/core.hpp +++ b/include/boost/unordered/detail/foa/core.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -158,6 +159,7 @@ static constexpr std::size_t default_bucket_count=0; #if defined(BOOST_UNORDERED_SSE2) +template class IntegralWrapper> struct group15 { static constexpr int N=15; @@ -168,7 +170,11 @@ struct group15 alignas(16) unsigned char storage[N+1]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0}; }; - inline void initialize(){m=_mm_setzero_si128();} + inline void initialize() + { + _mm_store_si128( + reinterpret_cast<__m128i*>(m),_mm_setzero_si128()); + } inline void set(std::size_t pos,std::size_t hash) { @@ -200,13 +206,14 @@ struct group15 static inline void reset(unsigned char* pc) { - *pc=available_; + *reinterpret_cast(pc)=available_; } inline int match(std::size_t hash)const { + auto w=_mm_load_si128(reinterpret_cast(m)); return _mm_movemask_epi8( - _mm_cmpeq_epi8(m,_mm_set1_epi32(match_word(hash))))&0x7FFF; + _mm_cmpeq_epi8(w,_mm_set1_epi32(match_word(hash))))&0x7FFF; } inline bool is_not_overflowed(std::size_t hash)const @@ -218,12 +225,7 @@ struct group15 inline void mark_overflow(std::size_t hash) { -#if BOOST_WORKAROUND(BOOST_GCC,>=50000 && BOOST_GCC<60000) - overflow()=static_cast( - overflow()|static_cast(1<<(hash%8))); -#else overflow()|=static_cast(1<<(hash%8)); -#endif } static inline bool maybe_caused_overflow(unsigned char* pc) @@ -235,13 +237,14 @@ struct group15 inline int match_available()const { + auto w=_mm_load_si128(reinterpret_cast(m)); return _mm_movemask_epi8( - _mm_cmpeq_epi8(m,_mm_setzero_si128()))&0x7FFF; + _mm_cmpeq_epi8(w,_mm_setzero_si128()))&0x7FFF; } static inline bool is_occupied(unsigned char* pc)noexcept { - return *pc!=available_; + return *reinterpret_cast(pc)!=available_; } inline int match_occupied()const @@ -255,6 +258,9 @@ struct group15 } private: + using slot_type=IntegralWrapper; + BOOST_STATIC_ASSERT(sizeof(slot_type)==1); + static constexpr unsigned char available_=0, sentinel_=1; @@ -304,31 +310,32 @@ private: return narrow_cast(match_word(hash)); } - inline unsigned char& at(std::size_t pos) + inline slot_type& at(std::size_t pos) { - return reinterpret_cast(&m)[pos]; + return m[pos]; } - inline unsigned char at(std::size_t pos)const + inline const slot_type& at(std::size_t pos)const { - return reinterpret_cast(&m)[pos]; + return m[pos]; } - inline unsigned char& overflow() + inline slot_type& overflow() { return at(N); } - inline unsigned char overflow()const + inline const slot_type& overflow()const { return at(N); } - alignas(16) __m128i m; + alignas(16) slot_type m[16]; }; #elif defined(BOOST_UNORDERED_LITTLE_ENDIAN_NEON) +template class IntegralWrapper> struct group15 { static constexpr int N=15; @@ -339,7 +346,10 @@ struct group15 alignas(16) unsigned char storage[N+1]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0}; }; - inline void initialize(){m=vdupq_n_s8(0);} + inline void initialize() + { + vst1q_u8(reinterpret_cast(m),vdupq_n_s8(0)); + } inline void set(std::size_t pos,std::size_t hash) { @@ -360,7 +370,7 @@ struct group15 static inline bool is_sentinel(unsigned char* pc)noexcept { - return *pc==sentinel_; + return *reinterpret_cast(pc)==sentinel_; } inline void reset(std::size_t pos) @@ -371,13 +381,14 @@ struct group15 static inline void reset(unsigned char* pc) { - *pc=available_; + *reinterpret_cast(pc)=available_; } inline int match(std::size_t hash)const { return simde_mm_movemask_epi8(vceqq_s8( - m,vdupq_n_s8(static_cast(reduced_hash(hash)))))&0x7FFF; + vld1q_u8(reinterpret_cast(m)), + vdupq_n_s8(static_cast(reduced_hash(hash)))))&0x7FFF; } inline bool is_not_overflowed(std::size_t hash)const @@ -401,18 +412,21 @@ struct group15 inline int match_available()const { - return simde_mm_movemask_epi8(vceqq_s8(m,vdupq_n_s8(0)))&0x7FFF; + return simde_mm_movemask_epi8(vceqq_s8( + vld1q_u8(reinterpret_cast(m)), + vdupq_n_s8(0)))&0x7FFF; } static inline bool is_occupied(unsigned char* pc)noexcept { - return *pc!=available_; + return *reinterpret_cast(pc)!=available_; } inline int match_occupied()const { - return simde_mm_movemask_epi8( - vcgtq_u8(vreinterpretq_u8_s8(m),vdupq_n_u8(0)))&0x7FFF; + return simde_mm_movemask_epi8(vcgtq_u8( + vreinterpretq_u8_s8(vld1q_u8(reinterpret_cast(m))), + vdupq_n_u8(0)))&0x7FFF; } inline int match_really_occupied()const /* excluding sentinel */ @@ -421,6 +435,9 @@ struct group15 } private: + using slot_type=IntegralWrapper; + BOOST_STATIC_ASSERT(sizeof(slot_type)==1); + static constexpr unsigned char available_=0, sentinel_=1; @@ -473,31 +490,32 @@ private: #endif } - inline unsigned char& at(std::size_t pos) + inline slot_type& at(std::size_t pos) { - return reinterpret_cast(&m)[pos]; + return m[pos]; } - inline unsigned char at(std::size_t pos)const + inline const slot_type& at(std::size_t pos)const { - return reinterpret_cast(&m)[pos]; + return m[pos]; } - inline unsigned char& overflow() + inline slot_type& overflow() { return at(N); } - inline unsigned char overflow()const + inline const slot_type& overflow()const { return at(N); } - alignas(16) int8x16_t m; + alignas(16) slot_type m[16]; }; #else /* non-SIMD */ +template class IntegralWrapper> struct group15 { static constexpr int N=15; @@ -590,6 +608,9 @@ struct group15 } private: + using word_type=IntegralWrapper; + BOOST_STATIC_ASSERT(sizeof(slot_type)==1); + static constexpr unsigned char available_=0, sentinel_=1; @@ -624,7 +645,7 @@ private: set_impl(m[1],pos,n>>4); } - static inline void set_impl(boost::uint64_t& x,std::size_t pos,std::size_t n) + static inline void set_impl(word_type& x,std::size_t pos,std::size_t n) { static constexpr boost::uint64_t mask[]= { @@ -670,7 +691,7 @@ private: return y&0x7FFF; } - alignas(16) boost::uint64_t m[2]; + alignas(16) word_type m[2]; }; #endif @@ -1064,7 +1085,10 @@ struct try_emplace_args_t{}; #pragma warning(disable:4702) #endif -template +template< + typename TypePolicy,typename Group,typename Hash,typename Pred, + typename Allocator +> class #if defined(_MSC_VER)&&_MSC_FULL_VER>=190023918 @@ -1075,7 +1099,7 @@ table_core:empty_value,empty_value,empty_value { public: using type_policy=TypePolicy; - using group_type=group15; + using group_type=Group; static constexpr auto N=group_type::N; using size_policy=pow2_size_policy; using prober=pow2_quadratic_prober; @@ -1103,8 +1127,8 @@ public: using locator=table_locator; table_core( - std::size_t n=0,const Hash& h_=Hash(),const Pred& pred_=Pred(), - const Allocator& al_=Allocator()): + std::size_t n=default_bucket_count,const Hash& h_=Hash(), + const Pred& pred_=Pred(),const Allocator& al_=Allocator()): hash_base{empty_init,h_},pred_base{empty_init,pred_}, allocator_base{empty_init,al_},size_{0},arrays(new_arrays(n)), ml{initial_max_load()} @@ -1576,7 +1600,8 @@ public: std::size_t ml; private: - template friend class table_core; + template + friend class table_core; using hash_base=empty_value; using pred_base=empty_value; diff --git a/include/boost/unordered/detail/foa/table.hpp b/include/boost/unordered/detail/foa/table.hpp index 508e8deb..3bc7a6d7 100644 --- a/include/boost/unordered/detail/foa/table.hpp +++ b/include/boost/unordered/detail/foa/table.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -26,6 +27,24 @@ namespace unordered{ namespace detail{ namespace foa{ +template +struct plain_integral +{ + operator Integral()const{return n;} + void operator=(Integral m){n=m;} + + void operator|=(Integral m) + { +#if BOOST_WORKAROUND(BOOST_GCC,>=50000 && BOOST_GCC<60000) + n=static_cast(n|m); +#else + n|=m; +#endif + } + + Integral n; +}; + template class table; @@ -81,7 +100,8 @@ public: const_iterator_cast_tag, const table_iterator& x): pc{x.pc},p{x.p}{} - inline reference operator*()const noexcept{return type_policy::value_from(*p);} + inline reference operator*()const noexcept + {return type_policy::value_from(*p);} inline pointer operator->()const noexcept {return std::addressof(type_policy::value_from(*p));} inline table_iterator& operator++()noexcept{increment();return *this;} @@ -181,7 +201,7 @@ union uninitialized_storage /* foa::table interface departs in a number of ways from that of C++ unordered * associative containers because it's not for end-user consumption - * (boost::unordered_[flat|node]_[map|set]) wrappers complete it as + * (boost::unordered_[flat|node]_[map|set] wrappers complete it as * appropriate). * * The table supports two main modes of operation: flat and node-based. In the @@ -208,9 +228,10 @@ union uninitialized_storage #include template -class table:table_core +class table:table_core,Hash,Pred,Allocator> { - using super=table_core; + using super= + table_core,Hash,Pred,Allocator>; using type_policy=typename super::type_policy; using group_type=typename super::group_type; using super::N; @@ -244,8 +265,8 @@ public: const_iterator>::type; table( - std::size_t n=0,const Hash& h_=Hash(),const Pred& pred_=Pred(), - const Allocator& al_=Allocator()): + std::size_t n=default_bucket_count,const Hash& h_=Hash(), + const Pred& pred_=Pred(),const Allocator& al_=Allocator()): super{n,h_,pred_,al_} {}