From 7415721f7d32be60bb874d10cb45d2e202471e71 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Sat, 25 Mar 2023 12:28:44 +0100 Subject: [PATCH] completed shared/exclusive visitation --- .../unordered/detail/foa/concurrent_table.hpp | 175 ++++++++++++------ 1 file changed, 114 insertions(+), 61 deletions(-) diff --git a/include/boost/unordered/detail/foa/concurrent_table.hpp b/include/boost/unordered/detail/foa/concurrent_table.hpp index d61b519d..d2fba57b 100644 --- a/include/boost/unordered/detail/foa/concurrent_table.hpp +++ b/include/boost/unordered/detail/foa/concurrent_table.hpp @@ -346,16 +346,16 @@ public: } template - BOOST_FORCEINLINE std::size_t visit(const Key& x,F f) + BOOST_FORCEINLINE std::size_t visit(const Key& x,F&& f) { auto lck=shared_access(); auto hash=this->hash_for(x); return unprotected_visit( - group_exclusive{},x,this->position_for(hash),hash,f); + group_exclusive{},x,this->position_for(hash),hash,std::forward(f)); } template - BOOST_FORCEINLINE std::size_t visit(const Key& x,F f)const + BOOST_FORCEINLINE std::size_t visit(const Key& x,F&& f)const { auto lck=shared_access(); auto hash=this->hash_for(x); @@ -364,40 +364,43 @@ public: [&](const value_type& v){f(v);}); } - template std::size_t visit_all(F f) + template std::size_t visit_all(F&& f) { - // TODO: replace with group-protected algorithm auto lck=shared_access(); std::size_t res=0; - this->for_all_elements([&](element_type* p){ + for_all_elements(group_exclusive{},[&](element_type* p){ f(type_policy::value_from(*p)); ++res; }); return res; } - template std::size_t visit_all(F f)const + template std::size_t visit_all(F&& f)const { - // TODO: replace with group-protected algorithm - return const_cast(this)-> - visit_all([&](const value_type& v){f(v);}); + auto lck=shared_access(); + std::size_t res=0; + for_all_elements(group_shared{},[&](element_type* p){ + f(const_cast(type_policy::value_from(*p))); + ++res; + }); + return res; } #if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS) template - void visit_all(ExecutionPolicy&& policy,F f) + void visit_all(ExecutionPolicy&& policy,F&& f) { auto lck=shared_access(); - for_all_elements_exec( + for_all_elements( group_exclusive{},std::forward(policy), [&](element_type* p){f(type_policy::value_from(*p));}); } template - void visit_all(ExecutionPolicy&& policy,F f)const + void visit_all(ExecutionPolicy&& policy,F&& f)const { auto lck=shared_access(); - for_all_elements_exec( + for_all_elements( group_shared{},std::forward(policy), [&](element_type* p) {f(const_cast(type_policy::value_from(*p)));}); @@ -444,35 +447,47 @@ public: } template - BOOST_FORCEINLINE bool try_emplace_or_visit(Key&& x,F f,Args&&... args) + BOOST_FORCEINLINE bool try_emplace_or_visit(Key&& x,F&& f,Args&&... args) { return emplace_or_visit_impl( - f,try_emplace_args_t{},std::forward(x),std::forward(args)...); + group_exclusive{},std::forward(f), + try_emplace_args_t{},std::forward(x),std::forward(args)...); } template - BOOST_FORCEINLINE bool emplace_or_visit(F f,Args&&... args) + BOOST_FORCEINLINE bool emplace_or_visit(F&& f,Args&&... args) { - return construct_and_emplace_or_visit(f,std::forward(args)...); + return construct_and_emplace_or_visit( + group_exclusive{},std::forward(f),std::forward(args)...); } template - BOOST_FORCEINLINE bool insert_or_visit(const init_type& x,F f) - {return emplace_or_visit_impl(f,x);} + BOOST_FORCEINLINE bool insert_or_visit(const init_type& x,F&& f) + { + return emplace_or_visit_impl(group_exclusive{},std::forward(f),x); + } template - BOOST_FORCEINLINE bool insert_or_visit(init_type&& x,F f) - {return emplace_or_visit_impl(f,std::move(x));} + BOOST_FORCEINLINE bool insert_or_visit(init_type&& x,F&& f) + { + return emplace_or_visit_impl( + group_exclusive{},std::forward(f),std::move(x)); + } /* typename=void tilts call ambiguities in favor of init_type */ template - BOOST_FORCEINLINE bool insert_or_visit(const value_type& x,F f) - {return emplace_or_visit_impl(f,x);} + BOOST_FORCEINLINE bool insert_or_visit(const value_type& x,F&& f) + { + return emplace_or_visit_impl(group_exclusive{},std::forward(f),x); + } template - BOOST_FORCEINLINE bool insert_or_visit(value_type&& x,F f) - {return emplace_or_visit_impl(f,std::move(x));} + BOOST_FORCEINLINE bool insert_or_visit(value_type&& x,F&& f) + { + return emplace_or_visit_impl( + group_exclusive{},std::forward(f),std::move(x)); + } template BOOST_FORCEINLINE std::size_t erase(Key&& x) @@ -481,7 +496,7 @@ public: } template - BOOST_FORCEINLINE auto erase_if(Key&& x,F f)->typename std::enable_if< + BOOST_FORCEINLINE auto erase_if(Key&& x,F&& f)->typename std::enable_if< !is_execution_policy::value,std::size_t>::type { auto lck=shared_access(); @@ -499,21 +514,27 @@ public: template std::size_t erase_if(F&& f) { - // TODO: replace with group-protected algorithm auto lck=shared_access(); - return super::erase_if_impl(std::forward(f)); + std::size_t s=size(); + for_all_elements( + group_exclusive{}, + [&,this](group_type* pg,unsigned int n,element_type* p){ + if(f(const_cast(type_policy::value_from(*p)))){ + super::erase(pg,n,p); + } + }); + return std::size_t(s-size()); } #if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS) template - auto erase_if(ExecutionPolicy&& policy,F f)->typename std::enable_if< + auto erase_if(ExecutionPolicy&& policy,F&& f)->typename std::enable_if< is_execution_policy::value,void>::type { auto lck=shared_access(); - for_all_elements_exec( + for_all_elements( group_exclusive{},std::forward(policy), - [&,this](group_type* pg,unsigned int n,element_type* p) - { + [&,this](group_type* pg,unsigned int n,element_type* p){ if(f(const_cast(type_policy::value_from(*p)))){ super::erase(pg,n,p); } @@ -538,13 +559,15 @@ public: template void merge(concurrent_table& x) { - // TODO: replace with group-protected algorithm // TODO: consider grabbing shared access on *this at this level - auto lck=x.shared_access(); // TODO: can deadlock if x1.merge(x2) while x2.merge(x1) - x.for_all_elements([&,this](group_type* pg,unsigned int n,element_type* p){ - erase_on_exit e{x,pg,n,p}; - if(!emplace_impl(type_policy::move(*p)))e.rollback(); - }); + // TODO: can deadlock if x1.merge(x2) while x2.merge(x1) + auto lck=x.shared_access(); + x.for_all_elements( + group_exclusive{}, + [&,this](group_type* pg,unsigned int n,element_type* p){ + erase_on_exit e{x,pg,n,p}; + if(!emplace_impl(type_policy::move(*p)))e.rollback(); + }); } template @@ -609,9 +632,9 @@ public: } template - friend std::size_t erase_if(concurrent_table& x,Predicate pr) + friend std::size_t erase_if(concurrent_table& x,Predicate&& pr) { - return x.erase_if(pr); + return x.erase_if(std::forward(pr)); } private: @@ -789,49 +812,52 @@ private: BOOST_FORCEINLINE bool construct_and_emplace(Args&&... args) { return construct_and_emplace_or_visit( - [](value_type&){},std::forward(args)...); + group_shared{},[](const value_type&){},std::forward(args)...); } - template - BOOST_FORCEINLINE bool construct_and_emplace_or_visit(F&& f,Args&&... args) + template + BOOST_FORCEINLINE bool construct_and_emplace_or_visit( + GroupAccessMode access_mode,F&& f,Args&&... args) { auto lck=shared_access(); auto x=alloc_make_insert_type( this->al(),std::forward(args)...); int res=unprotected_norehash_emplace_or_visit( - std::forward(f),type_policy::move(x.value())); + access_mode,std::forward(f),type_policy::move(x.value())); if(BOOST_LIKELY(res>=0))return res!=0; lck.unlock(); rehash_if_full(); return noinline_emplace_or_visit( - std::forward(f),type_policy::move(x.value())); + access_mode,std::forward(f),type_policy::move(x.value())); } template BOOST_FORCEINLINE bool emplace_impl(Args&&... args) { return emplace_or_visit_impl( - [](value_type&){},std::forward(args)...); + group_shared{},[](const value_type&){},std::forward(args)...); } - template - BOOST_NOINLINE bool noinline_emplace_or_visit(F&& f,Args&&... args) + template + BOOST_NOINLINE bool noinline_emplace_or_visit( + GroupAccessMode access_mode,F&& f,Args&&... args) { return emplace_or_visit_impl( - std::forward(f),std::forward(args)...); + access_mode,std::forward(f),std::forward(args)...); } - template - BOOST_FORCEINLINE bool emplace_or_visit_impl(F&& f,Args&&... args) + template + BOOST_FORCEINLINE bool emplace_or_visit_impl( + GroupAccessMode access_mode,F&& f,Args&&... args) { for(;;){ { auto lck=shared_access(); int res=unprotected_norehash_emplace_or_visit( - std::forward(f),std::forward(args)...); + access_mode,std::forward(f),std::forward(args)...); if(BOOST_LIKELY(res>=0))return res!=0; } rehash_if_full(); @@ -879,9 +905,10 @@ private: bool commit_=false; }; - template + template BOOST_FORCEINLINE int - unprotected_norehash_emplace_or_visit(F&& f,Args&&... args) + unprotected_norehash_emplace_or_visit( + GroupAccessMode access_mode,F&& f,Args&&... args) { const auto &k=this->key_from(std::forward(args)...); auto hash=this->hash_for(k); @@ -891,14 +918,14 @@ private: startover: boost::uint32_t counter=insert_counter(pos0); if(unprotected_visit( - group_exclusive{},k,pos0,hash,std::forward(f)))return 0; + access_mode,k,pos0,hash,std::forward(f)))return 0; reserve_size rsize(*this); if(BOOST_LIKELY(rsize.succeeded())){ for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){ auto pos=pb.get(); auto pg=this->arrays.groups+pos; - auto lck=exclusive_access(pos); + auto lck=access(group_exclusive{},pos); auto mask=pg->match_available(); if(BOOST_LIKELY(mask!=0)){ auto n=unchecked_countr_zero(mask); @@ -926,23 +953,49 @@ private: if(this->size_==this->ml)this->unchecked_rehash_for_growth(); } + template + auto for_all_elements(GroupAccessMode access_mode,F f) + ->decltype(f(nullptr),void()) + { + for_all_elements( + access_mode,[&](group_type*,unsigned int,element_type* p){f(p);}); + } + + template + auto for_all_elements(GroupAccessMode access_mode,F f) + ->decltype(f(nullptr,0,nullptr),void()) + { + auto p=this->arrays.elements; + if(!p)return; + for(auto pg=this->arrays.groups,last=pg+this->arrays.groups_size_mask+1; + pg!=last;++pg,p+=N){ + auto lck=access(access_mode,pg-this->arrays.groups); + auto mask=this->match_really_occupied(pg,last); + while(mask){ + auto n=unchecked_countr_zero(mask); + f(pg,n,p+n); + mask&=mask-1; + } + } + } + #if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS) template - auto for_all_elements_exec( + auto for_all_elements( GroupAccessMode access_mode,ExecutionPolicy&& policy,F f) ->decltype(f(nullptr),void()) { - for_all_elements_exec( + for_all_elements( access_mode,std::forward(policy), [&](group_type*,unsigned int,element_type* p){f(p);}); } template - auto for_all_elements_exec( + auto for_all_elements( GroupAccessMode access_mode,ExecutionPolicy&& policy,F f) ->decltype(f(nullptr,0,nullptr),void()) { - auto lck=shared_access(); + if(!this->arrays.elements)return; auto first=this->arrays.groups, last=first+this->arrays.groups_size_mask+1; std::for_each(std::forward(policy),first,last,