feedc0de/boost_unordered
1
0
Fork 0
forked from boostorg/unordered
Code Pull Requests Activity

Merge branch 'develop' into feature/detect_reentrancy

Browse Source
This commit is contained in:
joaquintides
2023-08-12 09:54:22 +02:00
committed by GitHub
parent 1979ce98a2 ee1974dc87
commit 66222eee6a
14 changed files with 897 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/boost/unordered/concurrent_flat_map.hpp
+61
View File
@@ -15,6 +15,7 @@
#include <boost/unordered/detail/foa/concurrent_table.hpp>
#include <boost/unordered/detail/foa/flat_map_types.hpp>
#include <boost/unordered/detail/type_traits.hpp>
#include <boost/unordered/unordered_flat_map_fwd.hpp>
#include <boost/container_hash/hash.hpp>
#include <boost/core/allocator_access.hpp>
@@ -84,6 +85,9 @@ namespace boost {
template <class Key2, class T2, class Hash2, class Pred2,
class Allocator2>
friend class concurrent_flat_map;
template <class Key2, class T2, class Hash2, class Pred2,
class Allocator2>
friend class unordered_flat_map;
using type_policy = detail::foa::flat_map_types<Key, T>;
@@ -223,6 +227,13 @@ namespace boost {
{
}
concurrent_flat_map(
unordered_flat_map<Key, T, Hash, Pred, Allocator>&& other)
: table_(std::move(other.table_))
{
}
~concurrent_flat_map() = default;
concurrent_flat_map& operator=(concurrent_flat_map const& rhs)
@@ -355,6 +366,56 @@ namespace boost {
}
#endif
template <class F> bool visit_while(F f)
{
BOOST_UNORDERED_STATIC_ASSERT_INVOCABLE(F)
return table_.visit_while(f);
}
template <class F> bool visit_while(F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
return table_.visit_while(f);
}
template <class F> bool cvisit_while(F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
return table_.cvisit_while(f);
}
#if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS)
template <class ExecPolicy, class F>
typename std::enable_if<detail::is_execution_policy<ExecPolicy>::value,
bool>::type
visit_while(ExecPolicy&& p, F f)
{
BOOST_UNORDERED_STATIC_ASSERT_INVOCABLE(F)
BOOST_UNORDERED_STATIC_ASSERT_EXEC_POLICY(ExecPolicy)
return table_.visit_while(p, f);
}
template <class ExecPolicy, class F>
typename std::enable_if<detail::is_execution_policy<ExecPolicy>::value,
bool>::type
visit_while(ExecPolicy&& p, F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
BOOST_UNORDERED_STATIC_ASSERT_EXEC_POLICY(ExecPolicy)
return table_.visit_while(p, f);
}
template <class ExecPolicy, class F>
typename std::enable_if<detail::is_execution_policy<ExecPolicy>::value,
bool>::type
cvisit_while(ExecPolicy&& p, F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
BOOST_UNORDERED_STATIC_ASSERT_EXEC_POLICY(ExecPolicy)
return table_.cvisit_while(p, f);
}
#endif
/// Modifiers
///
include/boost/unordered/detail/foa/concurrent_table.hpp
+171 -33
View File
@@ -253,7 +253,21 @@ struct concurrent_table_arrays:table_arrays<Value,Group,SizePolicy>
template<typename Allocator>
static concurrent_table_arrays new_(Allocator& al,std::size_t n)
{
concurrent_table_arrays arrays{super::new_(al,n),nullptr};
super x{super::new_(al,n)};
BOOST_TRY{
return new_group_access(al,x);
}
BOOST_CATCH(...){
super::delete_(al,x);
BOOST_RETHROW
}
BOOST_CATCH_END
}
template<typename Allocator>
static concurrent_table_arrays new_group_access(Allocator& al,const super& x)
{
concurrent_table_arrays arrays{x,nullptr};
if(!arrays.elements){
arrays.group_accesses=dummy_group_accesses<SizePolicy::min_size()>();
}
@@ -262,26 +276,26 @@ struct concurrent_table_arrays:table_arrays<Value,Group,SizePolicy>
typename boost::allocator_rebind<Allocator,group_access>::type;
using access_traits=boost::allocator_traits<access_alloc>;
BOOST_TRY{
auto aal=access_alloc(al);
arrays.group_accesses=boost::to_address(
access_traits::allocate(aal,arrays.groups_size_mask+1));
auto aal=access_alloc(al);
arrays.group_accesses=boost::to_address(
access_traits::allocate(aal,arrays.groups_size_mask+1));
for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){
::new (arrays.group_accesses+i) group_access();
}
for(std::size_t i=0;i<arrays.groups_size_mask+1;++i){
::new (arrays.group_accesses+i) group_access();
}
BOOST_CATCH(...){
super::delete_(al,arrays);
BOOST_RETHROW
}
BOOST_CATCH_END
}
return arrays;
}
template<typename Allocator>
static void delete_(Allocator& al,concurrent_table_arrays& arrays)noexcept
{
delete_group_access(al,arrays);
super::delete_(al,arrays);
}
template<typename Allocator>
static void delete_group_access(Allocator& al,concurrent_table_arrays& arrays)noexcept
{
if(arrays.elements){
using access_alloc=
@@ -295,7 +309,6 @@ struct concurrent_table_arrays:table_arrays<Value,Group,SizePolicy>
aal,pointer_traits::pointer_to(*arrays.group_accesses),
arrays.groups_size_mask+1);
}
super::delete_(al,arrays);
}
group_access *group_accesses;
@@ -308,7 +321,7 @@ struct atomic_size_control
atomic_size_control(std::size_t ml_,std::size_t size_):
pad0_{},ml{ml_},pad1_{},size{size_}{}
atomic_size_control(atomic_size_control& x):
atomic_size_control(const atomic_size_control& x):
pad0_{},ml{x.ml.load()},pad1_{},size{x.size.load()}{}
/* padding to avoid false sharing internally and with sorrounding data */
@@ -360,7 +373,7 @@ inline void swap(atomic_size_control& x,atomic_size_control& y)
* - Parallel versions of [c]visit_all(f) and erase_if(f) are provided based
* on C++17 stdlib parallel algorithms.
*
* Consult boost::unordered_flat_map docs for the full API reference.
* Consult boost::concurrent_flat_map docs for the full API reference.
* Heterogeneous lookup is suported by default, that is, without checking for
* any ::is_transparent typedefs --this checking is done by the wrapping
* containers.
@@ -392,6 +405,9 @@ inline void swap(atomic_size_control& x,atomic_size_control& y)
* over.
*/
template<typename,typename,typename,typename>
class table; /* concurrent/non-concurrent interop */
template <typename TypePolicy,typename Hash,typename Pred,typename Allocator>
using concurrent_table_core_impl=table_core<
TypePolicy,group15<atomic_integral>,concurrent_table_arrays,
@@ -413,10 +429,10 @@ class concurrent_table:
using group_type=typename super::group_type;
using super::N;
using prober=typename super::prober;
template<
typename TypePolicy2,typename Hash2,typename Pred2,typename Allocator2>
friend class concurrent_table;
using arrays_type=typename super::arrays_type;
using size_ctrl_type=typename super::size_ctrl_type;
using compatible_nonconcurrent_table=table<TypePolicy,Hash,Pred,Allocator>;
friend compatible_nonconcurrent_table;
public:
using key_type=typename super::key_type;
@@ -451,6 +467,21 @@ public:
concurrent_table(x,al_,x.exclusive_access()){}
concurrent_table(concurrent_table&& x,const Allocator& al_):
concurrent_table(std::move(x),al_,x.exclusive_access()){}
concurrent_table(compatible_nonconcurrent_table&& x):
super{
std::move(x.h()),std::move(x.pred()),std::move(x.al()),
arrays_type(arrays_type::new_group_access(
x.al(),
typename arrays_type::super{
x.arrays.groups_size_index,x.arrays.groups_size_mask,
reinterpret_cast<group_type*>(x.arrays.groups),
reinterpret_cast<value_type*>(x.arrays.elements)})),
size_ctrl_type{x.size_ctrl.ml,x.size_ctrl.size}}
{
x.empty_initialize();
}
~concurrent_table()=default;
concurrent_table& operator=(const concurrent_table& x)
@@ -540,6 +571,46 @@ public:
}
#endif
template<typename F> bool visit_while(F&& f)
{
return visit_while_impl(group_exclusive{},std::forward<F>(f));
}
template<typename F> bool visit_while(F&& f)const
{
return visit_while_impl(group_shared{},std::forward<F>(f));
}
template<typename F> bool cvisit_while(F&& f)const
{
return visit_while(std::forward<F>(f));
}
#if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS)
template<typename ExecutionPolicy,typename F>
bool visit_while(ExecutionPolicy&& policy,F&& f)
{
return visit_while_impl(
group_exclusive{},
std::forward<ExecutionPolicy>(policy),std::forward<F>(f));
}
template<typename ExecutionPolicy,typename F>
bool visit_while(ExecutionPolicy&& policy,F&& f)const
{
return visit_while_impl(
group_shared{},
std::forward<ExecutionPolicy>(policy),std::forward<F>(f));
}
template<typename ExecutionPolicy,typename F>
bool cvisit_while(ExecutionPolicy&& policy,F&& f)const
{
return visit_while(
std::forward<ExecutionPolicy>(policy),std::forward<F>(f));
}
#endif
bool empty()const noexcept{return size()==0;}
std::size_t size()const noexcept
@@ -836,6 +907,8 @@ public:
}
private:
template<typename,typename,typename,typename> friend class concurrent_table;
using mutex_type=rw_spinlock;
using multimutex_type=multimutex<mutex_type,128>; // TODO: adapt 128 to the machine
using shared_lock_guard=reentrancy_checked<shared_lock<mutex_type>>;
@@ -971,6 +1044,29 @@ private:
}
#endif
template<typename GroupAccessMode,typename F>
bool visit_while_impl(GroupAccessMode access_mode,F&& f)const
{
auto lck=shared_access();
return for_all_elements_while(access_mode,[&](element_type* p){
return f(cast_for(access_mode,type_policy::value_from(*p)));
});
}
#if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS)
template<typename GroupAccessMode,typename ExecutionPolicy,typename F>
bool visit_while_impl(
GroupAccessMode access_mode,ExecutionPolicy&& policy,F&& f)const
{
auto lck=shared_access();
return for_all_elements_while(
access_mode,std::forward<ExecutionPolicy>(policy),
[&](element_type* p){
return f(cast_for(access_mode,type_policy::value_from(*p)));
});
}
#endif
template<typename GroupAccessMode,typename Key,typename F>
BOOST_FORCEINLINE std::size_t unprotected_visit(
GroupAccessMode access_mode,
@@ -1254,19 +1350,38 @@ private:
template<typename GroupAccessMode,typename F>
auto for_all_elements(GroupAccessMode access_mode,F f)const
->decltype(f(nullptr,0,nullptr),void())
{
for_all_elements_while(
access_mode,[&](group_type* pg,unsigned int n,element_type* p)
{f(pg,n,p);return true;});
}
template<typename GroupAccessMode,typename F>
auto for_all_elements_while(GroupAccessMode access_mode,F f)const
->decltype(f(nullptr),bool())
{
return for_all_elements_while(
access_mode,[&](group_type*,unsigned int,element_type* p){return f(p);});
}
template<typename GroupAccessMode,typename F>
auto for_all_elements_while(GroupAccessMode access_mode,F f)const
->decltype(f(nullptr,0,nullptr),bool())
{
auto p=this->arrays.elements;
if(!p)return;
for(auto pg=this->arrays.groups,last=pg+this->arrays.groups_size_mask+1;
pg!=last;++pg,p+=N){
auto lck=access(access_mode,(std::size_t)(pg-this->arrays.groups));
auto mask=this->match_really_occupied(pg,last);
while(mask){
auto n=unchecked_countr_zero(mask);
f(pg,n,p+n);
mask&=mask-1;
if(p){
for(auto pg=this->arrays.groups,last=pg+this->arrays.groups_size_mask+1;
pg!=last;++pg,p+=N){
auto lck=access(access_mode,(std::size_t)(pg-this->arrays.groups));
auto mask=this->match_really_occupied(pg,last);
while(mask){
auto n=unchecked_countr_zero(mask);
if(!f(pg,n,p+n))return false;
mask&=mask-1;
}
}
}
return true;
}
#if defined(BOOST_UNORDERED_PARALLEL_ALGORITHMS)
@@ -1290,10 +1405,10 @@ private:
last=first+this->arrays.groups_size_mask+1;
std::for_each(std::forward<ExecutionPolicy>(policy),first,last,
[&,this](group_type& g){
std::size_t pos=static_cast<std::size_t>(&g-first);
auto p=this->arrays.elements+pos*N;
auto lck=access(access_mode,pos);
auto mask=this->match_really_occupied(&g,last);
auto pos=static_cast<std::size_t>(&g-first);
auto p=this->arrays.elements+pos*N;
auto lck=access(access_mode,pos);
auto mask=this->match_really_occupied(&g,last);
while(mask){
auto n=unchecked_countr_zero(mask);
f(&g,n,p+n);
@@ -1302,6 +1417,29 @@ private:
}
);
}
template<typename GroupAccessMode,typename ExecutionPolicy,typename F>
bool for_all_elements_while(
GroupAccessMode access_mode,ExecutionPolicy&& policy,F f)const
{
if(!this->arrays.elements)return true;
auto first=this->arrays.groups,
last=first+this->arrays.groups_size_mask+1;
return std::all_of(std::forward<ExecutionPolicy>(policy),first,last,
[&,this](group_type& g){
auto pos=static_cast<std::size_t>(&g-first);
auto p=this->arrays.elements+pos*N;
auto lck=access(access_mode,pos);
auto mask=this->match_really_occupied(&g,last);
while(mask){
auto n=unchecked_countr_zero(mask);
if(!f(p+n))return false;
mask&=mask-1;
}
return true;
}
);
}
#endif
static std::atomic<std::size_t> thread_counter;
include/boost/unordered/detail/foa/core.hpp
+24 -8
View File
@@ -1282,6 +1282,17 @@ public:
size_ctrl{initial_max_load(),0}
{}
/* bare transfer ctor for concurrent/non-concurrent interop */
table_core(
Hash&& h_,Pred&& pred_,Allocator&& al_,
const arrays_type& arrays_,const size_ctrl_type& size_ctrl_):
hash_base{empty_init,std::move(h_)},
pred_base{empty_init,std::move(pred_)},
allocator_base{empty_init,std::move(al_)},
arrays(arrays_),size_ctrl(size_ctrl_)
{}
table_core(const table_core& x):
table_core{x,alloc_traits::select_on_container_copy_construction(x.al())}{}
@@ -1290,14 +1301,11 @@ public:
std::is_nothrow_move_constructible<Hash>::value&&
std::is_nothrow_move_constructible<Pred>::value&&
std::is_nothrow_move_constructible<Allocator>::value):
hash_base{empty_init,std::move(x.h())},
pred_base{empty_init,std::move(x.pred())},
allocator_base{empty_init,std::move(x.al())},
arrays(x.arrays),size_ctrl(x.size_ctrl)
table_core{
std::move(x.h()),std::move(x.pred()),std::move(x.al()),
x.arrays,x.size_ctrl}
{
x.arrays=x.new_arrays(0);
x.size_ctrl.ml=x.initial_max_load();
x.size_ctrl.size=0;
x.empty_initialize();
}
table_core(const table_core& x,const Allocator& al_):
@@ -1336,6 +1344,13 @@ public:
delete_arrays(arrays);
}
void empty_initialize()noexcept
{
arrays=new_arrays(0);
size_ctrl.ml=initial_max_load();
size_ctrl.size=0;
}
table_core& operator=(const table_core& x)
{
BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED(Hash, Pred)
@@ -1804,7 +1819,8 @@ private:
pred_base{empty_init,std::move(pred_)},
allocator_base{empty_init,al_},arrays(new_arrays(0)),
size_ctrl{initial_max_load(),0}
{}
{
}
arrays_type new_arrays(std::size_t n)
{
include/boost/unordered/detail/foa/node_map_types.hpp
+1
View File
@@ -6,6 +6,7 @@
#define BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_TYPES_HPP
#include <boost/core/allocator_access.hpp>
#include <boost/core/no_exceptions_support.hpp>
#include <boost/core/pointer_traits.hpp>
namespace boost {
include/boost/unordered/detail/foa/node_set_types.hpp
+1
View File
@@ -6,6 +6,7 @@
#define BOOST_UNORDERED_DETAIL_FOA_NODE_SET_TYPES_HPP
#include <boost/core/allocator_access.hpp>
#include <boost/core/no_exceptions_support.hpp>
#include <boost/core/pointer_traits.hpp>
namespace boost {
include/boost/unordered/detail/foa/table.hpp
+26
View File
@@ -264,6 +264,9 @@ private:
* checking is done by boost::unordered_(flat|node)_(map|set).
*/
template<typename,typename,typename,typename>
class concurrent_table; /* concurrent/non-concurrent interop */
template <typename TypePolicy,typename Hash,typename Pred,typename Allocator>
using table_core_impl=
table_core<TypePolicy,group15<plain_integral>,table_arrays,
@@ -284,7 +287,12 @@ class table:table_core_impl<TypePolicy,Hash,Pred,Allocator>
using group_type=typename super::group_type;
using super::N;
using prober=typename super::prober;
using arrays_type=typename super::arrays_type;
using size_ctrl_type=typename super::size_ctrl_type;
using locator=typename super::locator;
using compatible_concurrent_table=
concurrent_table<TypePolicy,Hash,Pred,Allocator>;
friend compatible_concurrent_table;
public:
using key_type=typename super::key_type;
@@ -323,6 +331,8 @@ public:
table(table&& x)=default;
table(const table& x,const Allocator& al_):super{x,al_}{}
table(table&& x,const Allocator& al_):super{std::move(x),al_}{}
table(compatible_concurrent_table&& x):
table(std::move(x),x.exclusive_access()){}
~table()=default;
table& operator=(const table& x)=default;
@@ -496,6 +506,22 @@ public:
friend bool operator!=(const table& x,const table& y){return !(x==y);}
private:
template<typename ExclusiveLockGuard>
table(compatible_concurrent_table&& x,ExclusiveLockGuard):
super{
std::move(x.h()),std::move(x.pred()),std::move(x.al()),
arrays_type{
x.arrays.groups_size_index,x.arrays.groups_size_mask,
reinterpret_cast<group_type*>(x.arrays.groups),
reinterpret_cast<value_type*>(x.arrays.elements)},
size_ctrl_type{
x.size_ctrl.ml,x.size_ctrl.size}}
{
compatible_concurrent_table::arrays_type::delete_group_access(
this->al(),x.arrays);
x.empty_initialize();
}
struct erase_on_exit
{
erase_on_exit(table& x_,const_iterator it_):x{x_},it{it_}{}
include/boost/unordered/unordered_flat_map.hpp
+11
View File
@@ -10,6 +10,7 @@
#pragma once
#endif
#include <boost/unordered/concurrent_flat_map_fwd.hpp>
#include <boost/unordered/detail/foa/flat_map_types.hpp>
#include <boost/unordered/detail/foa/table.hpp>
#include <boost/unordered/detail/type_traits.hpp>
@@ -36,6 +37,10 @@ namespace boost {
template <class Key, class T, class Hash, class KeyEqual, class Allocator>
class unordered_flat_map
{
template <class Key2, class T2, class Hash2, class Pred2,
class Allocator2>
friend class concurrent_flat_map;
using map_types = detail::foa::flat_map_types<Key, T>;
using table_type = detail::foa::table<map_types, Hash, KeyEqual,
@@ -173,6 +178,12 @@ namespace boost {
{
}
unordered_flat_map(
concurrent_flat_map<Key, T, Hash, KeyEqual, Allocator>&& other)
: table_(std::move(other.table_))
{
}
~unordered_flat_map() = default;
unordered_flat_map& operator=(unordered_flat_map const& other)