diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 93036768..85bbc701 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -161,6 +161,7 @@ static const std::size_t default_bucket_count = 0; struct group15 { static constexpr int N=15; + static constexpr bool regular_layout=true; struct dummy_group_type { @@ -186,6 +187,11 @@ struct group15 return at(pos)==sentinel_; } + static inline bool is_sentinel(unsigned char* pc)noexcept + { + return *pc==sentinel_; + } + inline void reset(std::size_t pos) { BOOST_ASSERT(pos friend class table; table_iterator(Group* pg,std::size_t n,const table_element_type* p_): - pc{reinterpret_cast(const_cast(pg))+n}, + pc{reinterpret_cast(const_cast(pg))+n}, p{const_cast(p_)} {} - inline std::size_t rebase() noexcept - { - std::size_t off=reinterpret_cast(pc)%sizeof(Group); - pc-=off; - return off; - } - inline void increment()noexcept { - std::size_t n0=rebase(); + increment(std::integral_constant{}); + } - int mask=(reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); + inline void increment(std::true_type /* regular layout */)noexcept + { + for(;;){ + ++p; + if(reinterpret_cast(pc)%sizeof(group_type)==N-1){ + pc+=sizeof(group_type)-(N-1); + break; + } + ++pc; + if(!group_type::is_occupied(pc))continue; + if(BOOST_UNLIKELY(group_type::is_sentinel(pc)))p=nullptr; + return; + } + + for(;;){ + int mask=reinterpret_cast(pc)->match_occupied(); + if(mask!=0){ + auto n=unchecked_countr_zero(mask); + if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ + p=nullptr; + } + else{ + pc+=n; + p+=n; + } + return; + } + pc+=sizeof(group_type); + p+=N; + } + } + + inline void increment(std::false_type /* interleaved */)noexcept + { + std::size_t n0=reinterpret_cast(pc)%sizeof(group_type); + pc-=n0; + + int mask=( + reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); if(!mask){ do{ - pc+=sizeof(Group); - p+=Group::N; + pc+=sizeof(group_type); + p+=N; } - while((mask=reinterpret_cast(pc)->match_occupied())==0); + while((mask=reinterpret_cast(pc)->match_occupied())==0); } auto n=unchecked_countr_zero(mask); - if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ + if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ p=nullptr; } else{