|
|
|
@@ -50,6 +50,13 @@
|
|
|
|
|
#include <execution>
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
#include <array>
|
|
|
|
|
#include <algorithm>
|
|
|
|
|
#include <vector>
|
|
|
|
|
#include <iostream>
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
namespace boost{
|
|
|
|
|
namespace unordered{
|
|
|
|
|
namespace detail{
|
|
|
|
@@ -83,6 +90,8 @@ public:
|
|
|
|
|
cache_aligned_array(const cache_aligned_array&)=delete;
|
|
|
|
|
cache_aligned_array& operator=(const cache_aligned_array&)=delete;
|
|
|
|
|
|
|
|
|
|
constexpr std::size_t size()const noexcept{return N;}
|
|
|
|
|
|
|
|
|
|
T& operator[](std::size_t pos)noexcept{return *data(pos);}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
@@ -127,18 +136,16 @@ template<typename Mutex>
|
|
|
|
|
class shared_lock
|
|
|
|
|
{
|
|
|
|
|
public:
|
|
|
|
|
shared_lock(Mutex& m_)noexcept:m(m_){m.lock_shared();}
|
|
|
|
|
~shared_lock()noexcept{if(owns)m.unlock_shared();}
|
|
|
|
|
shared_lock(Mutex& m)noexcept:pm(&m){pm->lock_shared();}
|
|
|
|
|
shared_lock(shared_lock&& x)noexcept:pm(x.pm){x.pm=nullptr;x.owns=false;}
|
|
|
|
|
~shared_lock()noexcept{if(owns&&pm)pm->unlock_shared();}
|
|
|
|
|
|
|
|
|
|
/* not used but VS in pre-C++17 mode needs to see it for RVO */
|
|
|
|
|
shared_lock(const shared_lock&);
|
|
|
|
|
|
|
|
|
|
void lock(){BOOST_ASSERT(!owns);m.lock_shared();owns=true;}
|
|
|
|
|
void unlock(){BOOST_ASSERT(owns);m.unlock_shared();owns=false;}
|
|
|
|
|
void lock(){BOOST_ASSERT(!owns&&pm);pm->lock_shared();owns=true;}
|
|
|
|
|
void unlock(){BOOST_ASSERT(owns&&pm);pm->unlock_shared();owns=false;}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
Mutex &m;
|
|
|
|
|
bool owns=true;
|
|
|
|
|
Mutex* pm;
|
|
|
|
|
bool owns=true;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/* VS in pre-C++17 mode can't implement RVO for std::lock_guard due to
|
|
|
|
@@ -149,14 +156,12 @@ template<typename Mutex>
|
|
|
|
|
class lock_guard
|
|
|
|
|
{
|
|
|
|
|
public:
|
|
|
|
|
lock_guard(Mutex& m_)noexcept:m(m_){m.lock();}
|
|
|
|
|
~lock_guard()noexcept{m.unlock();}
|
|
|
|
|
|
|
|
|
|
/* not used but VS in pre-C++17 mode needs to see it for RVO */
|
|
|
|
|
lock_guard(const lock_guard&);
|
|
|
|
|
lock_guard(Mutex& m)noexcept:pm(&m){pm->lock();}
|
|
|
|
|
lock_guard(lock_guard&& x)noexcept:pm(x.pm){x.pm=nullptr;}
|
|
|
|
|
~lock_guard()noexcept{if(pm)pm->unlock();}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
Mutex &m;
|
|
|
|
|
Mutex* pm;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/* inspired by boost/multi_index/detail/scoped_bilock.hpp */
|
|
|
|
@@ -198,6 +203,17 @@ private:
|
|
|
|
|
template<typename Integral>
|
|
|
|
|
struct atomic_integral
|
|
|
|
|
{
|
|
|
|
|
#if 0&&defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
operator Integral()const{return n.load(std::memory_order_acquire);}
|
|
|
|
|
void operator=(Integral m){n.store(m,std::memory_order_release);}
|
|
|
|
|
void operator|=(Integral m){n.fetch_or(m);}
|
|
|
|
|
void operator&=(Integral m){n.fetch_and(m);}
|
|
|
|
|
|
|
|
|
|
atomic_integral& operator=(atomic_integral const& rhs) {
|
|
|
|
|
n.store(rhs.n.load(std::memory_order_acquire),std::memory_order_release);
|
|
|
|
|
return *this;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
operator Integral()const{return n.load(std::memory_order_relaxed);}
|
|
|
|
|
void operator=(Integral m){n.store(m,std::memory_order_relaxed);}
|
|
|
|
|
void operator|=(Integral m){n.fetch_or(m,std::memory_order_relaxed);}
|
|
|
|
@@ -207,6 +223,7 @@ struct atomic_integral
|
|
|
|
|
n.store(rhs.n.load(std::memory_order_relaxed),std::memory_order_relaxed);
|
|
|
|
|
return *this;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
std::atomic<Integral> n;
|
|
|
|
|
};
|
|
|
|
@@ -216,6 +233,17 @@ struct atomic_integral
|
|
|
|
|
* unprotected_norehash_emplace_or_visit).
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
struct group_access
|
|
|
|
|
{
|
|
|
|
|
using insert_counter_type=std::atomic<boost::uint32_t>;
|
|
|
|
|
|
|
|
|
|
insert_counter_type& insert_counter(){return icnt;}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
insert_counter_type icnt{0};
|
|
|
|
|
};
|
|
|
|
|
#else
|
|
|
|
|
struct group_access
|
|
|
|
|
{
|
|
|
|
|
using mutex_type=rw_spinlock;
|
|
|
|
@@ -231,6 +259,7 @@ private:
|
|
|
|
|
mutex_type m;
|
|
|
|
|
insert_counter_type cnt{0};
|
|
|
|
|
};
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
template<std::size_t Size>
|
|
|
|
|
group_access* dummy_group_accesses()
|
|
|
|
@@ -297,6 +326,16 @@ struct concurrent_table_arrays:table_arrays<Value,Group,SizePolicy,Allocator>
|
|
|
|
|
for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){
|
|
|
|
|
::new (arrays.group_accesses()+i) group_access();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
// initialize element access counters
|
|
|
|
|
|
|
|
|
|
static constexpr auto N=super::N;
|
|
|
|
|
|
|
|
|
|
for(std::size_t i=0;i<(arrays.groups_size_mask+1)*N-1;++i){
|
|
|
|
|
arrays.elements()[i].access=0;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void set_group_access(
|
|
|
|
@@ -480,8 +519,12 @@ public:
|
|
|
|
|
concurrent_table(
|
|
|
|
|
std::size_t n=default_bucket_count,const Hash& h_=Hash(),
|
|
|
|
|
const Pred& pred_=Pred(),const Allocator& al_=Allocator()):
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
super{(std::max)(n,std::size_t(1)),h_,pred_,al_} // TODO: won't work with n==0
|
|
|
|
|
#else
|
|
|
|
|
super{n,h_,pred_,al_}
|
|
|
|
|
{}
|
|
|
|
|
#endif
|
|
|
|
|
{}
|
|
|
|
|
|
|
|
|
|
concurrent_table(const concurrent_table& x):
|
|
|
|
|
concurrent_table(x,x.exclusive_access()){}
|
|
|
|
@@ -515,7 +558,19 @@ public:
|
|
|
|
|
concurrent_table(std::move(x),x.make_empty_arrays())
|
|
|
|
|
{}
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
~concurrent_table(){
|
|
|
|
|
std::cout
|
|
|
|
|
<<"version: 2024/10/21 12:15; "
|
|
|
|
|
<<"lf: "<<(double)size()/capacity()<<"; "
|
|
|
|
|
<<"size: "<<size()<<", "
|
|
|
|
|
<<"capacity: "<<capacity()<<"; "
|
|
|
|
|
<<"rehashes: "<<rehashes<<"; "
|
|
|
|
|
<<"max probe:"<<max_probe<<"\n";
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
~concurrent_table()=default;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
concurrent_table& operator=(const concurrent_table& x)
|
|
|
|
|
{
|
|
|
|
@@ -668,10 +723,21 @@ public:
|
|
|
|
|
|
|
|
|
|
bool empty()const noexcept{return size()==0;}
|
|
|
|
|
|
|
|
|
|
std::size_t size()const noexcept
|
|
|
|
|
auto size()const noexcept
|
|
|
|
|
{
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
using ssize_t=std::make_signed<std::size_t>::type;
|
|
|
|
|
|
|
|
|
|
auto lck=exclusive_access();
|
|
|
|
|
ssize_t res=static_cast<ssize_t>(this->size_ctrl.size);
|
|
|
|
|
for(const auto& sc:local_size_ctrls){
|
|
|
|
|
res+=sc.size;
|
|
|
|
|
}
|
|
|
|
|
return res;
|
|
|
|
|
#else
|
|
|
|
|
auto lck=shared_access();
|
|
|
|
|
return unprotected_size();
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
using super::max_size;
|
|
|
|
@@ -799,6 +865,30 @@ public:
|
|
|
|
|
return erase_if(x,[](const value_type&){return true;});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
template<typename Key,typename F>
|
|
|
|
|
BOOST_FORCEINLINE auto erase_if(const Key& x,F&& f)->typename std::enable_if<
|
|
|
|
|
!is_execution_policy<Key>::value,std::size_t>::type
|
|
|
|
|
{
|
|
|
|
|
auto lck=shared_access();
|
|
|
|
|
auto hash=this->hash_for(x);
|
|
|
|
|
std::size_t res=0;
|
|
|
|
|
unprotected_internal_visit(
|
|
|
|
|
group_exclusive{},x,this->position_for(hash),hash,
|
|
|
|
|
[&,this](group_type* pg,unsigned int n,element_type* p)
|
|
|
|
|
{
|
|
|
|
|
if(f(cast_for(group_shared{},type_policy::value_from(*p)))){
|
|
|
|
|
pg->reset(n);
|
|
|
|
|
auto& sc=local_size_ctrl();
|
|
|
|
|
sc.size.fetch_sub(1,std::memory_order_relaxed);
|
|
|
|
|
sc.mcos.fetch_add(
|
|
|
|
|
!pg->is_not_overflowed(hash),std::memory_order_relaxed);
|
|
|
|
|
res=1;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
template<typename Key,typename F>
|
|
|
|
|
BOOST_FORCEINLINE auto erase_if(const Key& x,F&& f)->typename std::enable_if<
|
|
|
|
|
!is_execution_policy<Key>::value,std::size_t>::type
|
|
|
|
@@ -817,6 +907,7 @@ public:
|
|
|
|
|
});
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
template<typename F>
|
|
|
|
|
std::size_t erase_if(F&& f)
|
|
|
|
@@ -970,9 +1061,16 @@ private:
|
|
|
|
|
using exclusive_lock_guard=reentrancy_checked<lock_guard<multimutex_type>>;
|
|
|
|
|
using exclusive_bilock_guard=
|
|
|
|
|
reentrancy_bichecked<scoped_bilock<multimutex_type>>;
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
struct group_shared_lock_guard{};
|
|
|
|
|
struct group_exclusive_lock_guard{};
|
|
|
|
|
using group_insert_counter_type=typename group_access::insert_counter_type;
|
|
|
|
|
#else
|
|
|
|
|
using group_shared_lock_guard=typename group_access::shared_lock_guard;
|
|
|
|
|
using group_exclusive_lock_guard=typename group_access::exclusive_lock_guard;
|
|
|
|
|
using group_insert_counter_type=typename group_access::insert_counter_type;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
concurrent_table(const concurrent_table& x,exclusive_lock_guard):
|
|
|
|
|
super{x}{}
|
|
|
|
@@ -985,9 +1083,15 @@ private:
|
|
|
|
|
concurrent_table&& x,const Allocator& al_,exclusive_lock_guard):
|
|
|
|
|
super{std::move(x),al_}{}
|
|
|
|
|
|
|
|
|
|
static inline std::size_t thread_id()
|
|
|
|
|
{
|
|
|
|
|
thread_local auto id=(++thread_counter);
|
|
|
|
|
return id;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
inline shared_lock_guard shared_access()const
|
|
|
|
|
{
|
|
|
|
|
thread_local auto id=(++thread_counter)%mutexes.size();
|
|
|
|
|
thread_local auto id=thread_id()%mutexes.size();
|
|
|
|
|
|
|
|
|
|
return shared_lock_guard{this,mutexes[id]};
|
|
|
|
|
}
|
|
|
|
@@ -1018,13 +1122,21 @@ private:
|
|
|
|
|
|
|
|
|
|
inline group_shared_lock_guard access(group_shared,std::size_t pos)const
|
|
|
|
|
{
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
return {};
|
|
|
|
|
#else
|
|
|
|
|
return this->arrays.group_accesses()[pos].shared_access();
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
inline group_exclusive_lock_guard access(
|
|
|
|
|
group_exclusive,std::size_t pos)const
|
|
|
|
|
{
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
return {};
|
|
|
|
|
#else
|
|
|
|
|
return this->arrays.group_accesses()[pos].exclusive_access();
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
inline group_insert_counter_type& insert_counter(std::size_t pos)const
|
|
|
|
@@ -1154,12 +1266,118 @@ private:
|
|
|
|
|
{f(cast_for(access_mode,type_policy::value_from(*p)));});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* check occupation with previous unsynced match */
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
static bool is_occupied(group_type* pg,std::size_t pos)
|
|
|
|
|
{
|
|
|
|
|
return pg->is_occupied(pos);
|
|
|
|
|
//return true;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
static bool is_occupied(group_type* pg,std::size_t pos)
|
|
|
|
|
{
|
|
|
|
|
return pg->is_occupied(pos);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_MSVC)
|
|
|
|
|
/* warning: forcing value to bool 'true' or 'false' in bool(pred()...) */
|
|
|
|
|
#pragma warning(push)
|
|
|
|
|
#pragma warning(disable:4800)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
template<typename F>
|
|
|
|
|
BOOST_FORCEINLINE auto load_access(element_type* p,F f)const
|
|
|
|
|
->std::pair<decltype(f()),boost::uint32_t>
|
|
|
|
|
{
|
|
|
|
|
auto& acnt=p->access;
|
|
|
|
|
for(;;){
|
|
|
|
|
auto n=
|
|
|
|
|
acnt.load(std::memory_order_acquire)&
|
|
|
|
|
~boost::uint32_t(1);
|
|
|
|
|
|
|
|
|
|
auto res=f();
|
|
|
|
|
|
|
|
|
|
std::atomic_thread_fence(std::memory_order_acquire);
|
|
|
|
|
if(BOOST_LIKELY(acnt.load(std::memory_order_relaxed)==n))return {res,n};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
template<typename F>
|
|
|
|
|
BOOST_FORCEINLINE bool save_access(element_type* p,boost::uint32_t n,F f)const
|
|
|
|
|
{
|
|
|
|
|
auto& acnt=p->access;
|
|
|
|
|
if(!acnt.compare_exchange_strong(n,n+1,std::memory_order_acq_rel)){
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
std::atomic_thread_fence(std::memory_order_release);
|
|
|
|
|
|
|
|
|
|
f();
|
|
|
|
|
|
|
|
|
|
acnt.store(n+2,std::memory_order_release);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
template<typename F>
|
|
|
|
|
BOOST_FORCEINLINE void save_access(element_type* p,F f)const /* no previous load cnt */
|
|
|
|
|
{
|
|
|
|
|
auto &acnt=p->access;
|
|
|
|
|
for(;;){
|
|
|
|
|
auto n=
|
|
|
|
|
acnt.load(std::memory_order_acquire)&
|
|
|
|
|
~boost::uint32_t(1);
|
|
|
|
|
//if(n%2==1)continue;
|
|
|
|
|
|
|
|
|
|
if(save_access(p,n,f))return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
template<typename GroupAccessMode,typename Key,typename F>
|
|
|
|
|
BOOST_FORCEINLINE std::size_t unprotected_internal_visit(
|
|
|
|
|
GroupAccessMode access_mode,
|
|
|
|
|
const Key& x,std::size_t pos0,std::size_t hash,F&& f)const
|
|
|
|
|
{
|
|
|
|
|
startover:
|
|
|
|
|
prober pb(pos0);
|
|
|
|
|
do{
|
|
|
|
|
auto pos=pb.get();
|
|
|
|
|
auto pg=this->arrays.groups()+pos;
|
|
|
|
|
auto mask=pg->match(hash);
|
|
|
|
|
if(mask){
|
|
|
|
|
auto p=this->arrays.elements()+pos*N;
|
|
|
|
|
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
|
|
|
|
|
do{
|
|
|
|
|
auto n=unchecked_countr_zero(mask);
|
|
|
|
|
auto [pr,acnt]=load_access(p+n,[&]{
|
|
|
|
|
return std::make_pair(is_occupied(pg,n),p[n].value);
|
|
|
|
|
});
|
|
|
|
|
auto [occupied,v]=pr;
|
|
|
|
|
|
|
|
|
|
if(BOOST_LIKELY(occupied&&this->pred()(x,this->key_from(v)))){
|
|
|
|
|
if constexpr(std::is_same<GroupAccessMode,group_exclusive>::value){
|
|
|
|
|
if(BOOST_UNLIKELY(!save_access(p+n,acnt,[&]{f(pg,n,p+n);}))){
|
|
|
|
|
goto startover;
|
|
|
|
|
}
|
|
|
|
|
return 1;
|
|
|
|
|
}else{
|
|
|
|
|
element_type e{v};
|
|
|
|
|
f(pg,n,&e);
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
mask&=mask-1;
|
|
|
|
|
}while(mask);
|
|
|
|
|
}
|
|
|
|
|
if(BOOST_LIKELY(pg->is_not_overflowed(hash))){
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
while(BOOST_LIKELY(pb.next(this->arrays.groups_size_mask)));
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
template<typename GroupAccessMode,typename Key,typename F>
|
|
|
|
|
BOOST_FORCEINLINE std::size_t unprotected_internal_visit(
|
|
|
|
|
GroupAccessMode access_mode,
|
|
|
|
@@ -1191,6 +1409,7 @@ private:
|
|
|
|
|
while(BOOST_LIKELY(pb.next(this->arrays.groups_size_mask)));
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
template<typename GroupAccessMode,typename FwdIterator,typename F>
|
|
|
|
|
BOOST_FORCEINLINE std::size_t unprotected_bulk_visit(
|
|
|
|
@@ -1236,7 +1455,7 @@ private:
|
|
|
|
|
do{
|
|
|
|
|
auto n=unchecked_countr_zero(mask);
|
|
|
|
|
if(BOOST_LIKELY(
|
|
|
|
|
pg->is_occupied(n)&&
|
|
|
|
|
is_occupied(pg,n)&&
|
|
|
|
|
bool(this->pred()(*it,this->key_from(p[n]))))){
|
|
|
|
|
f(cast_for(access_mode,type_policy::value_from(p[n])));
|
|
|
|
|
++res;
|
|
|
|
@@ -1436,6 +1655,94 @@ private:
|
|
|
|
|
bool commit_=false;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
struct latch_free_reserve_slot
|
|
|
|
|
{
|
|
|
|
|
latch_free_reserve_slot(group_type* pg,std::size_t pos):
|
|
|
|
|
pc{reinterpret_cast<std::atomic<unsigned char>*>(pg)+pos}
|
|
|
|
|
{
|
|
|
|
|
unsigned char expected=0;
|
|
|
|
|
succeeded_=pc->compare_exchange_weak(
|
|
|
|
|
expected,1,std::memory_order_relaxed,std::memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
~latch_free_reserve_slot()
|
|
|
|
|
{
|
|
|
|
|
if(succeeded_&&!commit_)pc->store(0,std::memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool succeeded()const{return succeeded_;}
|
|
|
|
|
|
|
|
|
|
void commit(){commit_=true;}
|
|
|
|
|
|
|
|
|
|
std::atomic<unsigned char>* pc;
|
|
|
|
|
bool succeeded_;
|
|
|
|
|
bool commit_=false;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct assign_insert_counter_on_exit
|
|
|
|
|
{
|
|
|
|
|
~assign_insert_counter_on_exit()
|
|
|
|
|
{
|
|
|
|
|
counter.store(x,std::memory_order_release);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
group_insert_counter_type &counter;
|
|
|
|
|
boost::uint32_t x;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
template<typename GroupAccessMode,typename F,typename... Args>
|
|
|
|
|
BOOST_FORCEINLINE int
|
|
|
|
|
unprotected_norehash_emplace_or_visit(
|
|
|
|
|
GroupAccessMode access_mode,F&& f,Args&&... args)
|
|
|
|
|
{
|
|
|
|
|
const auto &k=this->key_from(std::forward<Args>(args)...);
|
|
|
|
|
auto hash=this->hash_for(k);
|
|
|
|
|
auto pos0=this->position_for(hash);
|
|
|
|
|
|
|
|
|
|
startover:
|
|
|
|
|
boost::uint32_t counter=0;
|
|
|
|
|
do{
|
|
|
|
|
counter=insert_counter(pos0).load(std::memory_order_acquire);
|
|
|
|
|
}
|
|
|
|
|
while(BOOST_UNLIKELY(counter%2==1));
|
|
|
|
|
|
|
|
|
|
if(unprotected_visit(
|
|
|
|
|
access_mode,k,pos0,hash,std::forward<F>(f)))return 0;
|
|
|
|
|
|
|
|
|
|
std::size_t pbn=max_probe;
|
|
|
|
|
for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){
|
|
|
|
|
auto pos=pb.get();
|
|
|
|
|
auto p=this->arrays.elements()+pos*N;
|
|
|
|
|
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
|
|
|
|
|
auto pg=this->arrays.groups()+pos;
|
|
|
|
|
auto mask=pg->match_available();
|
|
|
|
|
if(BOOST_LIKELY(mask!=0)){
|
|
|
|
|
auto n=unchecked_countr_zero(mask);
|
|
|
|
|
auto [ocuppied,acnt]=load_access(p+n,[&]{return is_occupied(pg,n);});
|
|
|
|
|
if(ocuppied)goto startover;
|
|
|
|
|
|
|
|
|
|
auto res=false;
|
|
|
|
|
if(!save_access(p+n,acnt,[&]{
|
|
|
|
|
if(insert_counter(pos0).compare_exchange_strong(
|
|
|
|
|
counter,counter+1,std::memory_order_relaxed)){
|
|
|
|
|
assign_insert_counter_on_exit a{insert_counter(pos0),counter+2};
|
|
|
|
|
this->construct_element(p+n,std::forward<Args>(args)...);
|
|
|
|
|
pg->set(n,hash);
|
|
|
|
|
res=true;
|
|
|
|
|
}
|
|
|
|
|
})||!res)goto startover;
|
|
|
|
|
|
|
|
|
|
auto& sc=local_size_ctrl();
|
|
|
|
|
sc.size.fetch_add(1,std::memory_order_relaxed);
|
|
|
|
|
sc.mcos.fetch_sub(!pg->is_not_overflowed(hash),std::memory_order_relaxed);
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
if(!pbn--)return -1;
|
|
|
|
|
pg->mark_overflow(hash);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
template<typename GroupAccessMode,typename F,typename... Args>
|
|
|
|
|
BOOST_FORCEINLINE int
|
|
|
|
|
unprotected_norehash_emplace_or_visit(
|
|
|
|
@@ -1477,13 +1784,30 @@ private:
|
|
|
|
|
else return -1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
void rehash_if_full()
|
|
|
|
|
{
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
auto lck=shared_access();
|
|
|
|
|
auto p=calculate_size_ctrl();
|
|
|
|
|
lck.unlock();
|
|
|
|
|
if(p.first>=p.second){ // NB >=
|
|
|
|
|
auto lck=exclusive_access();
|
|
|
|
|
update_size_ctrl();
|
|
|
|
|
++rehashes;
|
|
|
|
|
this->unchecked_rehash_for_growth();
|
|
|
|
|
max_probe=default_max_probe;
|
|
|
|
|
}
|
|
|
|
|
else{
|
|
|
|
|
++max_probe;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
auto lck=exclusive_access();
|
|
|
|
|
if(this->size_ctrl.size==this->size_ctrl.ml){
|
|
|
|
|
this->unchecked_rehash_for_growth();
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
template<typename GroupAccessMode,typename F>
|
|
|
|
@@ -1724,6 +2048,62 @@ private:
|
|
|
|
|
|
|
|
|
|
static std::atomic<std::size_t> thread_counter;
|
|
|
|
|
mutable multimutex_type mutexes;
|
|
|
|
|
|
|
|
|
|
#if defined(BOOST_UNORDERED_LATCH_FREE)
|
|
|
|
|
struct alignas(64) local_size_ctrl_type
|
|
|
|
|
{
|
|
|
|
|
using ssize_t=std::make_signed<std::size_t>::type;
|
|
|
|
|
|
|
|
|
|
std::atomic<ssize_t> size=0;
|
|
|
|
|
std::atomic<ssize_t> mcos=0;
|
|
|
|
|
};
|
|
|
|
|
static constexpr std::size_t default_max_probe=3;
|
|
|
|
|
|
|
|
|
|
mutable std::array<local_size_ctrl_type,128> local_size_ctrls;
|
|
|
|
|
std::atomic<std::size_t> max_probe=default_max_probe;
|
|
|
|
|
std::size_t rehashes=0;
|
|
|
|
|
//unsigned char paddd[64];
|
|
|
|
|
|
|
|
|
|
local_size_ctrl_type& local_size_ctrl()const
|
|
|
|
|
{
|
|
|
|
|
return local_size_ctrls[thread_id()%local_size_ctrls.size()];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::pair<std::size_t,std::size_t> calculate_size_ctrl()
|
|
|
|
|
{
|
|
|
|
|
using ssize_t=std::make_signed<std::size_t>::type;
|
|
|
|
|
|
|
|
|
|
ssize_t ssize=0,smcos=0;
|
|
|
|
|
for(const auto& sc:local_size_ctrls){
|
|
|
|
|
ssize+=sc.size.load(std::memory_order_relaxed);
|
|
|
|
|
smcos+=sc.mcos.load(std::memory_order_relaxed);
|
|
|
|
|
}
|
|
|
|
|
std::size_t size_=this->size_ctrl.size.load(std::memory_order_relaxed),
|
|
|
|
|
ml_=this->size_ctrl.ml.load(std::memory_order_relaxed);
|
|
|
|
|
size_+=ssize;
|
|
|
|
|
if(ssize_t(ml_)>=smcos)ml_-=smcos;
|
|
|
|
|
else ml_=0;
|
|
|
|
|
auto max_ml=super::initial_max_load();
|
|
|
|
|
if(ml_>max_ml)ml_=max_ml;
|
|
|
|
|
return {size_,ml_};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void update_size_ctrl()
|
|
|
|
|
{
|
|
|
|
|
using ssize_t=std::make_signed<std::size_t>::type;
|
|
|
|
|
|
|
|
|
|
ssize_t ssize=0,smcos=0;
|
|
|
|
|
for(auto& sc:local_size_ctrls){
|
|
|
|
|
ssize+=sc.size.exchange(0);
|
|
|
|
|
smcos+=sc.mcos.exchange(0);
|
|
|
|
|
}
|
|
|
|
|
this->size_ctrl.size+=ssize;
|
|
|
|
|
if(ssize_t(this->size_ctrl.ml)>=smcos)this->size_ctrl.ml-=smcos;
|
|
|
|
|
else this->size_ctrl.ml=0;
|
|
|
|
|
auto max_ml=super::initial_max_load();
|
|
|
|
|
if(this->size_ctrl.ml>max_ml)this->size_ctrl.ml=max_ml;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
template<typename T,typename H,typename P,typename A>
|
|
|
|
|