Compare commits

..

74 Commits

Author SHA1 Message Date
joaquintides 5f7f6be571 included missing header 2024-10-21 12:43:20 +02:00
joaquintides a4a5a3e127 fixed trivial bug in unused function 2024-03-14 09:34:38 +01:00
joaquintides 2f63dc4dd8 implmented element-level ll/sc access 2024-03-08 20:08:33 +01:00
joaquintides a08e86d985 used two-stage ll/sc 2024-02-14 20:18:17 +01:00
joaquintides b22c09700d fixed erase_if on sentinels, implemented one-access non-constant visitation 2024-02-12 13:41:20 +01:00
joaquintides bad10788ff strengthened erase, relaxed metadata ops 2024-02-12 12:02:07 +01:00
joaquintides a444cfb1ae implemented latch-free element save 2024-02-10 20:51:09 +01:00
joaquintides db779b949e added missing synchronization 2024-02-09 19:40:45 +01:00
joaquintides 83ac0d59cc weakened many atomic ops 2024-02-09 19:08:33 +01:00
joaquintides a05b55270a reenabled info logging 2024-02-09 12:43:04 +01:00
joaquintides 77a0f6bb74 implemented latch-free element load 2024-02-09 12:33:41 +01:00
joaquintides 687a446784 used parlay epoch framework 2024-02-03 17:37:08 +01:00
joaquintides 1c48ce8194 removed node recycling 2024-02-02 19:09:39 +01:00
joaquintides 1e9970d150 reverted 2024-02-02 12:07:58 +01:00
joaquintides f10a9ab76d supressed wasted nodes report, relaxed metadata ops 2024-02-02 11:31:13 +01:00
joaquintides 83a81d6b1d relaxed epoch bumping 2024-02-01 20:04:00 +01:00
joaquintides 9b5545e30c avoided unnecessary epoch bumps 2024-01-29 13:12:47 +01:00
joaquintides 9e8d69c08a made garbage vectors bigger 2024-01-29 11:53:38 +01:00
joaquintides 99cee83c57 monitored wasted nodes 2024-01-27 11:53:48 +01:00
joaquintides 363ef61cf0 monitor garbage vector full condition 2024-01-26 11:15:28 +01:00
joaquintides b67aaf3558 cacheline-aligned garbage vectors 2024-01-22 09:50:23 +01:00
joaquintides 1bae403d58 added node recycling 2024-01-21 18:53:33 +01:00
joaquintides 4021c276fe switched to node-based layout 2024-01-20 11:52:19 +01:00
joaquintides d7f7792ee3 reduced contention on rehash_if_full, reduced min_for_epoch_bump 2024-01-13 13:12:07 +01:00
joaquintides 1d7165f759 incremented min_for_epoch_bump 2024-01-08 18:52:15 +01:00
joaquintides 25fe5948a6 sped up gc frequency 2024-01-08 17:52:52 +01:00
joaquintides f55e6ac290 made gc thread local 2024-01-08 11:31:39 +01:00
joaquintides 78aaa8ea51 re-strengthened atomic ops, halved gc frequency again 2024-01-03 20:50:17 +01:00
joaquintides 07370c16e6 halved gc frequency 2024-01-03 17:46:43 +01:00
joaquintides 679faa51a9 re-enabled gc 2024-01-03 13:24:14 +01:00
joaquintides 628df0f500 disabled is_occupied 2024-01-03 12:48:37 +01:00
joaquintides 16eec4a9a7 relaxed atomic ops 2024-01-03 11:55:00 +01:00
joaquintides 3310fe061c temporarily disabled gc 2024-01-02 19:43:43 +01:00
joaquintides 8d3c924724 increased gc frequency 2024-01-02 12:01:17 +01:00
joaquintides d576363d09 streamlined update_size_ctrl, printed concurrent_table stats 2024-01-02 10:42:01 +01:00
joaquintides 0aab436707 protected visitation against spurious insertion sentinels 2023-12-27 13:34:34 +01:00
joaquintides 71e9d35ab2 made latch-free unprotected_norehash_emplace_or_visit exception safe 2023-12-26 19:08:51 +01:00
joaquintides f3f01e33f4 fined-tuned gc dimensions 2023-12-26 18:01:36 +01:00
joaquintides c861b082b3 re-strengthened atomic_integral (insert-visit sync) 2023-12-26 10:01:16 +01:00
joaquintides 1418b96fb2 capped size_ctrl.ml in update_size_ctrl 2023-12-25 13:02:57 +01:00
joaquintides a68b92d1cd * relaxed atomic_integral
* re-enabled gc
* optimized mco calculation
* added heuristic to *reduce* mco at insertion time
* fixed signedness bug at update_size_ctrl
2023-12-25 12:48:04 +01:00
joaquintides 71fb7636c5 prevented gc overcall, temporarily disabled gc 2023-12-20 19:02:49 +01:00
joaquintides 5808607c4f reverted 2023-12-19 12:42:11 +01:00
joaquintides e91f743530 temporarily restricted gc to one designated thread 2023-12-19 12:22:11 +01:00
joaquintides 13d6cc0a33 reduced insertion contention zone 2023-12-19 11:34:35 +01:00
joaquintides 4cd3cd5ac2 used pointers rather than positions in the garbage collector 2023-12-18 13:38:57 +01:00
joaquintides b55a13c6b6 simplified erasure 2023-12-18 13:25:14 +01:00
joaquintides 35138450c8 desynced size control 2023-12-16 11:55:23 +01:00
joaquintides ca7120d928 removed prior experiments and optimized insertion via two-step counter 2023-12-14 12:36:04 +01:00
joaquintides b866bba144 removed counter-based protection on insertion 2023-12-13 13:51:05 +01:00
joaquintides 6869f099a5 disabled insertion synchronization 2023-12-13 13:43:09 +01:00
joaquintides aaf553581f disabled size control 2023-12-13 13:35:10 +01:00
joaquintides 116d2807f4 temporarily disabled garbage collection 2023-12-13 12:44:24 +01:00
joaquintides 567cfe4988 reverted 2023-12-12 16:47:59 +01:00
joaquintides c346a513f0 temporarily removed size control 2023-12-12 16:34:27 +01:00
joaquintides 64614f1670 optimized non-visiting insertion 2023-12-12 13:35:48 +01:00
joaquintides 8ac326b39f synchronized construct_and_emplace 2023-12-12 10:32:00 +01:00
joaquintides 5c21fa6d81 implemented group_synchronized_shared mode for insertion 2023-12-11 20:56:30 +01:00
joaquintides b8d9605107 fine-tuned memory order for metadata manipulation 2023-12-11 19:43:05 +01:00
joaquintides b087f9fb53 strengthened memory order for metadata manipulation 2023-12-11 18:54:10 +01:00
joaquintides 901d652fd7 implemented garbage collection 2023-12-11 17:37:35 +01:00
joaquintides 0f2a2a0b0f added latch-free erasure 2023-12-05 12:20:40 +01:00
joaquintides 86856f7e6b made epoch mechanism oversubscription free 2023-12-04 14:05:36 +01:00
joaquintides 8bd1bcf5de gotten rid of wait_for_epochs in insertion 2023-12-04 11:26:35 +01:00
joaquintides d6a8a4462c reverted to epoch array of size 128 and to non-relaxed wait_for_epochs 2023-12-03 19:51:26 +01:00
joaquintides 7e495f4598 relaxed operation overlooked in prior commit 2023-12-03 13:08:52 +01:00
joaquintides 68b38901e7 relaxed wait_for_epochs 2023-12-03 13:03:39 +01:00
joaquintides 821a2bfb45 made epoch array smaller 2023-12-03 09:45:09 +01:00
joaquintides 5af735827f optimized epoch waiting 2023-12-02 20:52:20 +01:00
joaquintides 49ff2a63ea made insertion and constant visitation (but *not* erasure) optionally latch-free 2023-12-02 19:50:36 +01:00
joaquintides f493603f5c [skip ci] editorial 2023-12-01 12:22:48 +01:00
Peter Dimov 74ca1b0e74 Update test/CMakeLists.txt 2023-11-30 17:54:53 +02:00
Peter Dimov 7b16324869 Regenerate CMakeLists.txt 2023-11-30 17:40:41 +02:00
joaquintides 6680e86b38 [skip ci] typo 2023-11-25 09:49:24 +01:00
6 changed files with 409 additions and 28 deletions
+3 -6
View File
@@ -3,7 +3,7 @@
# Distributed under the Boost Software License, Version 1.0.
# https://www.boost.org/LICENSE_1_0.txt
cmake_minimum_required(VERSION 3.5...3.20)
cmake_minimum_required(VERSION 3.8...3.20)
project(boost_unordered VERSION "${BOOST_SUPERPROJECT_VERSION}" LANGUAGES CXX)
@@ -18,16 +18,13 @@ target_link_libraries(boost_unordered
Boost::config
Boost::container_hash
Boost::core
Boost::move
Boost::mp11
Boost::predef
Boost::preprocessor
Boost::static_assert
Boost::throw_exception
Boost::tuple
Boost::type_traits
)
target_compile_features(boost_unordered INTERFACE cxx_std_11)
if(BUILD_TESTING AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/CMakeLists.txt")
add_subdirectory(test)
+1 -1
View File
@@ -38,7 +38,7 @@ Users can then declare a hash function `Hash` as avalanching either by embedding
into the definition of `Hash`, or directly by specializing `hash_is_avalanching<Hash>` to a class with
an embedded compile-time constant `value` set to `true`.
xref:unordered_flat_set[`boost::unordered_flat_set`] and xref:unordered_flat_map[`boost::unordered_flat_map`]
Open-addressing and concurrent containers
use the provided hash function `Hash` as-is if `hash_is_avalanching<Hash>::value` is `true`; otherwise, they
implement a bit-mixing post-processing stage to increase the quality of hashing at the expense of
extra computational cost.
+1 -1
View File
@@ -152,7 +152,7 @@ performing container-wide operations such as swapping or assignment.
By using atomic operations to access the group metadata, lookup is (group-level)
lock-free up to the point where an actual comparison needs to be done with an element
that has been previously SIMD-matched: only then it's the group's spinlock used.
that has been previously SIMD-matched: only then is the group's spinlock used.
Insertion uses the following _optimistic algorithm_:
@@ -50,6 +50,13 @@
#include <execution>
#endif
#if defined(BOOST_UNORDERED_LATCH_FREE)
#include <array>
#include <algorithm>
#include <vector>
#include <iostream>
#endif
namespace boost{
namespace unordered{
namespace detail{
@@ -83,6 +90,8 @@ public:
cache_aligned_array(const cache_aligned_array&)=delete;
cache_aligned_array& operator=(const cache_aligned_array&)=delete;
constexpr std::size_t size()const noexcept{return N;}
T& operator[](std::size_t pos)noexcept{return *data(pos);}
private:
@@ -127,18 +136,16 @@ template<typename Mutex>
class shared_lock
{
public:
shared_lock(Mutex& m_)noexcept:m(m_){m.lock_shared();}
~shared_lock()noexcept{if(owns)m.unlock_shared();}
shared_lock(Mutex& m)noexcept:pm(&m){pm->lock_shared();}
shared_lock(shared_lock&& x)noexcept:pm(x.pm){x.pm=nullptr;x.owns=false;}
~shared_lock()noexcept{if(owns&&pm)pm->unlock_shared();}
/* not used but VS in pre-C++17 mode needs to see it for RVO */
shared_lock(const shared_lock&);
void lock(){BOOST_ASSERT(!owns);m.lock_shared();owns=true;}
void unlock(){BOOST_ASSERT(owns);m.unlock_shared();owns=false;}
void lock(){BOOST_ASSERT(!owns&&pm);pm->lock_shared();owns=true;}
void unlock(){BOOST_ASSERT(owns&&pm);pm->unlock_shared();owns=false;}
private:
Mutex &m;
bool owns=true;
Mutex* pm;
bool owns=true;
};
/* VS in pre-C++17 mode can't implement RVO for std::lock_guard due to
@@ -149,14 +156,12 @@ template<typename Mutex>
class lock_guard
{
public:
lock_guard(Mutex& m_)noexcept:m(m_){m.lock();}
~lock_guard()noexcept{m.unlock();}
/* not used but VS in pre-C++17 mode needs to see it for RVO */
lock_guard(const lock_guard&);
lock_guard(Mutex& m)noexcept:pm(&m){pm->lock();}
lock_guard(lock_guard&& x)noexcept:pm(x.pm){x.pm=nullptr;}
~lock_guard()noexcept{if(pm)pm->unlock();}
private:
Mutex &m;
Mutex* pm;
};
/* inspired by boost/multi_index/detail/scoped_bilock.hpp */
@@ -198,6 +203,17 @@ private:
template<typename Integral>
struct atomic_integral
{
#if 0&&defined(BOOST_UNORDERED_LATCH_FREE)
operator Integral()const{return n.load(std::memory_order_acquire);}
void operator=(Integral m){n.store(m,std::memory_order_release);}
void operator|=(Integral m){n.fetch_or(m);}
void operator&=(Integral m){n.fetch_and(m);}
atomic_integral& operator=(atomic_integral const& rhs) {
n.store(rhs.n.load(std::memory_order_acquire),std::memory_order_release);
return *this;
}
#else
operator Integral()const{return n.load(std::memory_order_relaxed);}
void operator=(Integral m){n.store(m,std::memory_order_relaxed);}
void operator|=(Integral m){n.fetch_or(m,std::memory_order_relaxed);}
@@ -207,6 +223,7 @@ struct atomic_integral
n.store(rhs.n.load(std::memory_order_relaxed),std::memory_order_relaxed);
return *this;
}
#endif
std::atomic<Integral> n;
};
@@ -216,6 +233,17 @@ struct atomic_integral
* unprotected_norehash_emplace_or_visit).
*/
#if defined(BOOST_UNORDERED_LATCH_FREE)
struct group_access
{
using insert_counter_type=std::atomic<boost::uint32_t>;
insert_counter_type& insert_counter(){return icnt;}
private:
insert_counter_type icnt{0};
};
#else
struct group_access
{
using mutex_type=rw_spinlock;
@@ -231,6 +259,7 @@ private:
mutex_type m;
insert_counter_type cnt{0};
};
#endif
template<std::size_t Size>
group_access* dummy_group_accesses()
@@ -297,6 +326,16 @@ struct concurrent_table_arrays:table_arrays<Value,Group,SizePolicy,Allocator>
for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){
::new (arrays.group_accesses()+i) group_access();
}
#if defined(BOOST_UNORDERED_LATCH_FREE)
// initialize element access counters
static constexpr auto N=super::N;
for(std::size_t i=0;i<(arrays.groups_size_mask+1)*N-1;++i){
arrays.elements()[i].access=0;
}
#endif
}
static void set_group_access(
@@ -480,8 +519,12 @@ public:
concurrent_table(
std::size_t n=default_bucket_count,const Hash& h_=Hash(),
const Pred& pred_=Pred(),const Allocator& al_=Allocator()):
#if defined(BOOST_UNORDERED_LATCH_FREE)
super{(std::max)(n,std::size_t(1)),h_,pred_,al_} // TODO: won't work with n==0
#else
super{n,h_,pred_,al_}
{}
#endif
{}
concurrent_table(const concurrent_table& x):
concurrent_table(x,x.exclusive_access()){}
@@ -515,7 +558,19 @@ public:
concurrent_table(std::move(x),x.make_empty_arrays())
{}
#if defined(BOOST_UNORDERED_LATCH_FREE)
~concurrent_table(){
std::cout
<<"version: 2024/10/21 12:15; "
<<"lf: "<<(double)size()/capacity()<<"; "
<<"size: "<<size()<<", "
<<"capacity: "<<capacity()<<"; "
<<"rehashes: "<<rehashes<<"; "
<<"max probe:"<<max_probe<<"\n";
}
#else
~concurrent_table()=default;
#endif
concurrent_table& operator=(const concurrent_table& x)
{
@@ -668,10 +723,21 @@ public:
bool empty()const noexcept{return size()==0;}
std::size_t size()const noexcept
auto size()const noexcept
{
#if defined(BOOST_UNORDERED_LATCH_FREE)
using ssize_t=std::make_signed<std::size_t>::type;
auto lck=exclusive_access();
ssize_t res=static_cast<ssize_t>(this->size_ctrl.size);
for(const auto& sc:local_size_ctrls){
res+=sc.size;
}
return res;
#else
auto lck=shared_access();
return unprotected_size();
#endif
}
using super::max_size;
@@ -799,6 +865,30 @@ public:
return erase_if(x,[](const value_type&){return true;});
}
#if defined(BOOST_UNORDERED_LATCH_FREE)
template<typename Key,typename F>
BOOST_FORCEINLINE auto erase_if(const Key& x,F&& f)->typename std::enable_if<
!is_execution_policy<Key>::value,std::size_t>::type
{
auto lck=shared_access();
auto hash=this->hash_for(x);
std::size_t res=0;
unprotected_internal_visit(
group_exclusive{},x,this->position_for(hash),hash,
[&,this](group_type* pg,unsigned int n,element_type* p)
{
if(f(cast_for(group_shared{},type_policy::value_from(*p)))){
pg->reset(n);
auto& sc=local_size_ctrl();
sc.size.fetch_sub(1,std::memory_order_relaxed);
sc.mcos.fetch_add(
!pg->is_not_overflowed(hash),std::memory_order_relaxed);
res=1;
}
});
return res;
}
#else
template<typename Key,typename F>
BOOST_FORCEINLINE auto erase_if(const Key& x,F&& f)->typename std::enable_if<
!is_execution_policy<Key>::value,std::size_t>::type
@@ -817,6 +907,7 @@ public:
});
return res;
}
#endif
template<typename F>
std::size_t erase_if(F&& f)
@@ -970,9 +1061,16 @@ private:
using exclusive_lock_guard=reentrancy_checked<lock_guard<multimutex_type>>;
using exclusive_bilock_guard=
reentrancy_bichecked<scoped_bilock<multimutex_type>>;
#if defined(BOOST_UNORDERED_LATCH_FREE)
struct group_shared_lock_guard{};
struct group_exclusive_lock_guard{};
using group_insert_counter_type=typename group_access::insert_counter_type;
#else
using group_shared_lock_guard=typename group_access::shared_lock_guard;
using group_exclusive_lock_guard=typename group_access::exclusive_lock_guard;
using group_insert_counter_type=typename group_access::insert_counter_type;
#endif
concurrent_table(const concurrent_table& x,exclusive_lock_guard):
super{x}{}
@@ -985,9 +1083,15 @@ private:
concurrent_table&& x,const Allocator& al_,exclusive_lock_guard):
super{std::move(x),al_}{}
static inline std::size_t thread_id()
{
thread_local auto id=(++thread_counter);
return id;
}
inline shared_lock_guard shared_access()const
{
thread_local auto id=(++thread_counter)%mutexes.size();
thread_local auto id=thread_id()%mutexes.size();
return shared_lock_guard{this,mutexes[id]};
}
@@ -1018,13 +1122,21 @@ private:
inline group_shared_lock_guard access(group_shared,std::size_t pos)const
{
#if defined(BOOST_UNORDERED_LATCH_FREE)
return {};
#else
return this->arrays.group_accesses()[pos].shared_access();
#endif
}
inline group_exclusive_lock_guard access(
group_exclusive,std::size_t pos)const
{
#if defined(BOOST_UNORDERED_LATCH_FREE)
return {};
#else
return this->arrays.group_accesses()[pos].exclusive_access();
#endif
}
inline group_insert_counter_type& insert_counter(std::size_t pos)const
@@ -1154,12 +1266,118 @@ private:
{f(cast_for(access_mode,type_policy::value_from(*p)));});
}
/* check occupation with previous unsynced match */
#if defined(BOOST_UNORDERED_LATCH_FREE)
static bool is_occupied(group_type* pg,std::size_t pos)
{
return pg->is_occupied(pos);
//return true;
}
#else
static bool is_occupied(group_type* pg,std::size_t pos)
{
return pg->is_occupied(pos);
}
#endif
#if defined(BOOST_MSVC)
/* warning: forcing value to bool 'true' or 'false' in bool(pred()...) */
#pragma warning(push)
#pragma warning(disable:4800)
#endif
#if defined(BOOST_UNORDERED_LATCH_FREE)
template<typename F>
BOOST_FORCEINLINE auto load_access(element_type* p,F f)const
->std::pair<decltype(f()),boost::uint32_t>
{
auto& acnt=p->access;
for(;;){
auto n=
acnt.load(std::memory_order_acquire)&
~boost::uint32_t(1);
auto res=f();
std::atomic_thread_fence(std::memory_order_acquire);
if(BOOST_LIKELY(acnt.load(std::memory_order_relaxed)==n))return {res,n};
}
}
template<typename F>
BOOST_FORCEINLINE bool save_access(element_type* p,boost::uint32_t n,F f)const
{
auto& acnt=p->access;
if(!acnt.compare_exchange_strong(n,n+1,std::memory_order_acq_rel)){
return false;
}
std::atomic_thread_fence(std::memory_order_release);
f();
acnt.store(n+2,std::memory_order_release);
return true;
}
template<typename F>
BOOST_FORCEINLINE void save_access(element_type* p,F f)const /* no previous load cnt */
{
auto &acnt=p->access;
for(;;){
auto n=
acnt.load(std::memory_order_acquire)&
~boost::uint32_t(1);
//if(n%2==1)continue;
if(save_access(p,n,f))return;
}
}
template<typename GroupAccessMode,typename Key,typename F>
BOOST_FORCEINLINE std::size_t unprotected_internal_visit(
GroupAccessMode access_mode,
const Key& x,std::size_t pos0,std::size_t hash,F&& f)const
{
startover:
prober pb(pos0);
do{
auto pos=pb.get();
auto pg=this->arrays.groups()+pos;
auto mask=pg->match(hash);
if(mask){
auto p=this->arrays.elements()+pos*N;
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
do{
auto n=unchecked_countr_zero(mask);
auto [pr,acnt]=load_access(p+n,[&]{
return std::make_pair(is_occupied(pg,n),p[n].value);
});
auto [occupied,v]=pr;
if(BOOST_LIKELY(occupied&&this->pred()(x,this->key_from(v)))){
if constexpr(std::is_same<GroupAccessMode,group_exclusive>::value){
if(BOOST_UNLIKELY(!save_access(p+n,acnt,[&]{f(pg,n,p+n);}))){
goto startover;
}
return 1;
}else{
element_type e{v};
f(pg,n,&e);
return 1;
}
}
mask&=mask-1;
}while(mask);
}
if(BOOST_LIKELY(pg->is_not_overflowed(hash))){
return 0;
}
}
while(BOOST_LIKELY(pb.next(this->arrays.groups_size_mask)));
return 0;
}
#else
template<typename GroupAccessMode,typename Key,typename F>
BOOST_FORCEINLINE std::size_t unprotected_internal_visit(
GroupAccessMode access_mode,
@@ -1191,6 +1409,7 @@ private:
while(BOOST_LIKELY(pb.next(this->arrays.groups_size_mask)));
return 0;
}
#endif
template<typename GroupAccessMode,typename FwdIterator,typename F>
BOOST_FORCEINLINE std::size_t unprotected_bulk_visit(
@@ -1236,7 +1455,7 @@ private:
do{
auto n=unchecked_countr_zero(mask);
if(BOOST_LIKELY(
pg->is_occupied(n)&&
is_occupied(pg,n)&&
bool(this->pred()(*it,this->key_from(p[n]))))){
f(cast_for(access_mode,type_policy::value_from(p[n])));
++res;
@@ -1436,6 +1655,94 @@ private:
bool commit_=false;
};
#if defined(BOOST_UNORDERED_LATCH_FREE)
struct latch_free_reserve_slot
{
latch_free_reserve_slot(group_type* pg,std::size_t pos):
pc{reinterpret_cast<std::atomic<unsigned char>*>(pg)+pos}
{
unsigned char expected=0;
succeeded_=pc->compare_exchange_weak(
expected,1,std::memory_order_relaxed,std::memory_order_relaxed);
}
~latch_free_reserve_slot()
{
if(succeeded_&&!commit_)pc->store(0,std::memory_order_relaxed);
}
bool succeeded()const{return succeeded_;}
void commit(){commit_=true;}
std::atomic<unsigned char>* pc;
bool succeeded_;
bool commit_=false;
};
struct assign_insert_counter_on_exit
{
~assign_insert_counter_on_exit()
{
counter.store(x,std::memory_order_release);
}
group_insert_counter_type &counter;
boost::uint32_t x;
};
template<typename GroupAccessMode,typename F,typename... Args>
BOOST_FORCEINLINE int
unprotected_norehash_emplace_or_visit(
GroupAccessMode access_mode,F&& f,Args&&... args)
{
const auto &k=this->key_from(std::forward<Args>(args)...);
auto hash=this->hash_for(k);
auto pos0=this->position_for(hash);
startover:
boost::uint32_t counter=0;
do{
counter=insert_counter(pos0).load(std::memory_order_acquire);
}
while(BOOST_UNLIKELY(counter%2==1));
if(unprotected_visit(
access_mode,k,pos0,hash,std::forward<F>(f)))return 0;
std::size_t pbn=max_probe;
for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){
auto pos=pb.get();
auto p=this->arrays.elements()+pos*N;
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
auto pg=this->arrays.groups()+pos;
auto mask=pg->match_available();
if(BOOST_LIKELY(mask!=0)){
auto n=unchecked_countr_zero(mask);
auto [ocuppied,acnt]=load_access(p+n,[&]{return is_occupied(pg,n);});
if(ocuppied)goto startover;
auto res=false;
if(!save_access(p+n,acnt,[&]{
if(insert_counter(pos0).compare_exchange_strong(
counter,counter+1,std::memory_order_relaxed)){
assign_insert_counter_on_exit a{insert_counter(pos0),counter+2};
this->construct_element(p+n,std::forward<Args>(args)...);
pg->set(n,hash);
res=true;
}
})||!res)goto startover;
auto& sc=local_size_ctrl();
sc.size.fetch_add(1,std::memory_order_relaxed);
sc.mcos.fetch_sub(!pg->is_not_overflowed(hash),std::memory_order_relaxed);
return 1;
}
if(!pbn--)return -1;
pg->mark_overflow(hash);
}
}
#else
template<typename GroupAccessMode,typename F,typename... Args>
BOOST_FORCEINLINE int
unprotected_norehash_emplace_or_visit(
@@ -1477,13 +1784,30 @@ private:
else return -1;
}
}
#endif
void rehash_if_full()
{
#if defined(BOOST_UNORDERED_LATCH_FREE)
auto lck=shared_access();
auto p=calculate_size_ctrl();
lck.unlock();
if(p.first>=p.second){ // NB >=
auto lck=exclusive_access();
update_size_ctrl();
++rehashes;
this->unchecked_rehash_for_growth();
max_probe=default_max_probe;
}
else{
++max_probe;
}
#else
auto lck=exclusive_access();
if(this->size_ctrl.size==this->size_ctrl.ml){
this->unchecked_rehash_for_growth();
}
#endif
}
template<typename GroupAccessMode,typename F>
@@ -1724,6 +2048,62 @@ private:
static std::atomic<std::size_t> thread_counter;
mutable multimutex_type mutexes;
#if defined(BOOST_UNORDERED_LATCH_FREE)
struct alignas(64) local_size_ctrl_type
{
using ssize_t=std::make_signed<std::size_t>::type;
std::atomic<ssize_t> size=0;
std::atomic<ssize_t> mcos=0;
};
static constexpr std::size_t default_max_probe=3;
mutable std::array<local_size_ctrl_type,128> local_size_ctrls;
std::atomic<std::size_t> max_probe=default_max_probe;
std::size_t rehashes=0;
//unsigned char paddd[64];
local_size_ctrl_type& local_size_ctrl()const
{
return local_size_ctrls[thread_id()%local_size_ctrls.size()];
}
std::pair<std::size_t,std::size_t> calculate_size_ctrl()
{
using ssize_t=std::make_signed<std::size_t>::type;
ssize_t ssize=0,smcos=0;
for(const auto& sc:local_size_ctrls){
ssize+=sc.size.load(std::memory_order_relaxed);
smcos+=sc.mcos.load(std::memory_order_relaxed);
}
std::size_t size_=this->size_ctrl.size.load(std::memory_order_relaxed),
ml_=this->size_ctrl.ml.load(std::memory_order_relaxed);
size_+=ssize;
if(ssize_t(ml_)>=smcos)ml_-=smcos;
else ml_=0;
auto max_ml=super::initial_max_load();
if(ml_>max_ml)ml_=max_ml;
return {size_,ml_};
}
void update_size_ctrl()
{
using ssize_t=std::make_signed<std::size_t>::type;
ssize_t ssize=0,smcos=0;
for(auto& sc:local_size_ctrls){
ssize+=sc.size.exchange(0);
smcos+=sc.mcos.exchange(0);
}
this->size_ctrl.size+=ssize;
if(ssize_t(this->size_ctrl.ml)>=smcos)this->size_ctrl.ml-=smcos;
else this->size_ctrl.ml=0;
auto max_ml=super::initial_max_load();
if(this->size_ctrl.ml>max_ml)this->size_ctrl.ml=max_ml;
}
#endif
};
template<typename T,typename H,typename P,typename A>
@@ -382,11 +382,13 @@ private:
return (int)word[narrow_cast<unsigned char>(hash)];
}
public:
inline static unsigned char reduced_hash(std::size_t hash)
{
return narrow_cast<unsigned char>(match_word(hash));
}
private:
inline slot_type& at(std::size_t pos)
{
return m[pos];
@@ -532,6 +534,7 @@ private:
#endif
}
public:
inline static unsigned char reduced_hash(std::size_t hash)
{
static constexpr unsigned char table[]={
@@ -556,6 +559,7 @@ private:
return table[(unsigned char)hash];
}
private:
/* Copied from
* https://github.com/simd-everywhere/simde/blob/master/simde/x86/
* sse2.h#L3763
+1 -1
View File
@@ -9,7 +9,7 @@ if(HAVE_BOOST_TEST)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
set(BOOST_TEST_LINK_LIBRARIES Boost::unordered Boost::core Boost::concept_check)
set(BOOST_TEST_LINK_LIBRARIES Boost::unordered Boost::core Boost::concept_check Boost::tuple)
function(fca_tests)
boost_test(PREFIX boost_unordered ${ARGN})