boostorg/unordered
1
0
Fork 1
mirror of https://github.com/boostorg/unordered.git synced 2025-07-31 11:57:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

implmented element-level ll/sc access

Browse Source
This commit is contained in:
joaquintides
2024-03-08 20:08:33 +01:00
parent a08e86d985
commit 2f63dc4dd8
1 changed files with 40 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
include/boost/unordered/detail/foa/concurrent_table.hpp
View File

@@ -235,14 +235,11 @@ struct atomic_integral
#if defined(BOOST_UNORDERED_LATCH_FREE) #if defined(BOOST_UNORDERED_LATCH_FREE)
struct group_access struct group_access
{ {
using access_counter_type=std::atomic<boost::uint32_t>;
using insert_counter_type=std::atomic<boost::uint32_t>; using insert_counter_type=std::atomic<boost::uint32_t>;
access_counter_type& access_counter(){return acnt;}
insert_counter_type& insert_counter(){return icnt;} insert_counter_type& insert_counter(){return icnt;}
private: private:
access_counter_type acnt{0};
insert_counter_type icnt{0}; insert_counter_type icnt{0};
}; };
#else #else
@@ -328,6 +325,16 @@ struct concurrent_table_arrays:table_arrays<Value,Group,SizePolicy,Allocator>
for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){ for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){
::new (arrays.group_accesses()+i) group_access(); ::new (arrays.group_accesses()+i) group_access();
} }
#if defined(BOOST_UNORDERED_LATCH_FREE)
// initialize element access counters
static constexpr auto N=super::N;
for(std::size_t i=0;i<(arrays.groups_size_mask+1)*N-1;++i){
arrays.elements()[i].access=0;
}
#endif
} }
static void set_group_access( static void set_group_access(
@@ -553,7 +560,7 @@ public:
#if defined(BOOST_UNORDERED_LATCH_FREE) #if defined(BOOST_UNORDERED_LATCH_FREE)
~concurrent_table(){ ~concurrent_table(){
std::cout std::cout
<<"version: 2024/02/14 20:10; " <<"version: 2024/03/08 20:10; "
<<"lf: "<<(double)size()/capacity()<<"; " <<"lf: "<<(double)size()/capacity()<<"; "
<<"size: "<<size()<<", " <<"size: "<<size()<<", "
<<"capacity: "<<capacity()<<"; " <<"capacity: "<<capacity()<<"; "
@@ -869,8 +876,7 @@ public:
group_exclusive{},x,this->position_for(hash),hash, group_exclusive{},x,this->position_for(hash),hash,
[&,this](group_type* pg,unsigned int n,element_type* p) [&,this](group_type* pg,unsigned int n,element_type* p)
{ {
if(/*is_occupied(pg,n)&&*/ if(f(cast_for(group_shared{},type_policy::value_from(*p)))){
f(cast_for(group_shared{},type_policy::value_from(*p)))){
pg->reset(n); pg->reset(n);
auto& sc=local_size_ctrl(); auto& sc=local_size_ctrl();
sc.size.fetch_sub(1,std::memory_order_relaxed); sc.size.fetch_sub(1,std::memory_order_relaxed);
@@ -1058,7 +1064,6 @@ private:
#if defined(BOOST_UNORDERED_LATCH_FREE) #if defined(BOOST_UNORDERED_LATCH_FREE)
struct group_shared_lock_guard{}; struct group_shared_lock_guard{};
struct group_exclusive_lock_guard{}; struct group_exclusive_lock_guard{};
using group_access_counter_type=typename group_access::access_counter_type;
using group_insert_counter_type=typename group_access::insert_counter_type; using group_insert_counter_type=typename group_access::insert_counter_type;
#else #else
using group_shared_lock_guard=typename group_access::shared_lock_guard; using group_shared_lock_guard=typename group_access::shared_lock_guard;
@@ -1133,13 +1138,6 @@ private:
#endif #endif
} }
#if defined(BOOST_UNORDERED_LATCH_FREE)
inline group_access_counter_type& access_counter(std::size_t pos)const
{
return this->arrays.group_accesses()[pos].access_counter();
}
#endif
inline group_insert_counter_type& insert_counter(std::size_t pos)const inline group_insert_counter_type& insert_counter(std::size_t pos)const
{ {
return this->arrays.group_accesses()[pos].insert_counter(); return this->arrays.group_accesses()[pos].insert_counter();
@@ -1290,10 +1288,10 @@ private:
#if defined(BOOST_UNORDERED_LATCH_FREE) #if defined(BOOST_UNORDERED_LATCH_FREE)
template<typename F> template<typename F>
BOOST_FORCEINLINE auto load_access(std::size_t pos,F f)const BOOST_FORCEINLINE auto load_access(element_type* p,F f)const
->std::pair<decltype(f()),boost::uint32_t> ->std::pair<decltype(f()),boost::uint32_t>
{ {
auto& acnt=access_counter(pos); auto& acnt=p->access;
for(;;){ for(;;){
auto n= auto n=
acnt.load(std::memory_order_acquire)& acnt.load(std::memory_order_acquire)&
@@ -1307,9 +1305,9 @@ private:
} }
template<typename F> template<typename F>
BOOST_FORCEINLINE bool save_access(std::size_t pos,boost::uint32_t n,F f)const BOOST_FORCEINLINE bool save_access(element_type* p,boost::uint32_t n,F f)const
{ {
auto& acnt=access_counter(pos); auto& acnt=p->access;
if(!acnt.compare_exchange_strong(n,n+1,std::memory_order_acq_rel)){ if(!acnt.compare_exchange_strong(n,n+1,std::memory_order_acq_rel)){
return false; return false;
} }
@@ -1322,9 +1320,9 @@ private:
} }
template<typename F> template<typename F>
BOOST_FORCEINLINE void save_access(std::size_t pos,F f)const /* no previous load cnt */ BOOST_FORCEINLINE void save_access(element_type* p,F f)const /* no previous load cnt */
{ {
auto &acnt=access_counter(pos); auto &acnt=p->access;
for(;;){ for(;;){
auto n= auto n=
acnt.load(std::memory_order_acquire)& acnt.load(std::memory_order_acquire)&
@@ -1351,48 +1349,23 @@ private:
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N); BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
do{ do{
auto n=unchecked_countr_zero(mask); auto n=unchecked_countr_zero(mask);
#if 1 auto [pr,acnt]=load_access(p+n,[&]{
auto [pr,acnt]=load_access(pos,[&]{ return std::make_pair(is_occupied(pg,n),p[n].value);
return std::make_pair(is_occupied(pg,n),*(p+n));
}); });
auto [occupied,e]=pr; auto [occupied,v]=pr;
if(!occupied)return 0;
if(BOOST_LIKELY(this->pred()(x,this->key_from(e)))){
if constexpr(std::is_same<GroupAccessMode,group_exclusive>::value){
// ALTERNATIVE: offline f(pg,n,&e)
if(!save_access(pos,acnt,[&]{f(pg,n,p+n);})){ if(BOOST_LIKELY(occupied&&this->pred()(x,this->key_from(v)))){
// SPECIFIC FOR erase: if the element changed, it can only be if constexpr(std::is_same<GroupAccessMode,group_exclusive>::value){
// bc someone erased it. if(BOOST_UNLIKELY(!save_access(p+n,acnt,[&]{f(pg,n,p+n);}))){
// WATCH OUT: only approximate as acnt controls 15 elements, goto startover;
// not just this one.
return 0;
} }
return 1; return 1;
}else{ }else{
element_type e{v};
f(pg,n,&e); f(pg,n,&e);
return 1; return 1;
} }
} }
#else
if constexpr(std::is_same<GroupAccessMode,group_exclusive>::value){
bool res=false;
save_access(pos,[&]{
if(BOOST_LIKELY(this->pred()(x,this->key_from(p[n])))){
f(pg,n,p+n);
res=true;
}
});
if(res)return 1;
}
else{
auto [e,acnt]=load_access(pos,[&]{return *(p+n);});
if(BOOST_LIKELY(this->pred()(x,this->key_from(e)))){
f(pg,n,&e);
return 1;
}
}
#endif
mask&=mask-1; mask&=mask-1;
}while(mask); }while(mask);
} }
@@ -1739,42 +1712,26 @@ private:
std::size_t pbn=max_probe; std::size_t pbn=max_probe;
for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){ for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){
auto pos=pb.get(); auto pos=pb.get();
auto p=this->arrays.elements()+pos*N;
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
auto pg=this->arrays.groups()+pos; auto pg=this->arrays.groups()+pos;
auto mask=pg->match_available(); auto mask=pg->match_available();
if(BOOST_LIKELY(mask!=0)){ if(BOOST_LIKELY(mask!=0)){
auto n=unchecked_countr_zero(mask); auto n=unchecked_countr_zero(mask);
//latch_free_reserve_slot rslot{pg,n}; auto [ocuppied,acnt]=load_access(p+n,[&]{return is_occupied(pg,n);});
//if(BOOST_UNLIKELY(!rslot.succeeded())){ if(ocuppied)goto startover;
// /* slot wasn't empty */
// goto startover;
//}
//if(BOOST_UNLIKELY(
// !insert_counter(pos0).compare_exchange_strong(
// counter,counter+1,std::memory_order_relaxed))){
// /* other thread inserted from pos0, need to start over */
// goto startover;
//}
{
//assign_insert_counter_on_exit a{insert_counter(pos0),counter+2};
auto [ocuppied,acnt]=load_access(pos,[&]{return is_occupied(pg,n);}); auto res=false;
if(ocuppied)goto startover; if(!save_access(p+n,acnt,[&]{
if(insert_counter(pos0).compare_exchange_strong(
counter,counter+1,std::memory_order_relaxed)){
assign_insert_counter_on_exit a{insert_counter(pos0),counter+2};
this->construct_element(p+n,std::forward<Args>(args)...);
pg->set(n,hash);
res=true;
}
})||!res)goto startover;
auto res=false;
if(!save_access(pos,acnt,[&]{
if(insert_counter(pos0).compare_exchange_strong(
counter,counter+1,std::memory_order_relaxed)){
assign_insert_counter_on_exit a{
insert_counter(pos0),counter+2};
this->construct_element(
this->arrays.elements()+pos*N+n,std::forward<Args>(args)...);
pg->set(n,hash);
res=true;
}
})||!res)goto startover;
}
//rslot.commit();
auto& sc=local_size_ctrl(); auto& sc=local_size_ctrl();
sc.size.fetch_add(1,std::memory_order_relaxed); sc.size.fetch_add(1,std::memory_order_relaxed);
sc.mcos.fetch_sub(!pg->is_not_overflowed(hash),std::memory_order_relaxed); sc.mcos.fetch_sub(!pg->is_not_overflowed(hash),std::memory_order_relaxed);