diff --git a/doc/unordered/changes.adoc b/doc/unordered/changes.adoc index 18e5be56..76a5f7fe 100644 --- a/doc/unordered/changes.adoc +++ b/doc/unordered/changes.adoc @@ -15,6 +15,8 @@ with serial and parallel variants. * Added debug mode mechanisms for detecting illegal reentrancies into a `boost::concurrent_flat_map` from user code. * Added Boost.Serialization support to all containers and their (non-local) iterator types. +* Added support for fancy pointers to open-addressing and concurrent containers. + This enables scenarios like the use of Boost.Interprocess allocators to construct containers in shared memory. == Release 1.83.0 - Major update diff --git a/doc/unordered/concurrent_flat_map.adoc b/doc/unordered/concurrent_flat_map.adoc index bcfc719e..a2b28c9c 100644 --- a/doc/unordered/concurrent_flat_map.adoc +++ b/doc/unordered/concurrent_flat_map.adoc @@ -90,9 +90,10 @@ namespace boost { const allocator_type& a); xref:#concurrent_flat_map_destructor[~concurrent_flat_map](); concurrent_flat_map& xref:#concurrent_flat_map_copy_assignment[operator++=++](const concurrent_flat_map& other); - concurrent_flat_map& xref:#concurrent_flat_map_move_assignment[operator++=++](concurrent_flat_map&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + concurrent_flat_map& xref:#concurrent_flat_map_move_assignment[operator++=++](concurrent_flat_map&& other) ++noexcept( + (boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value);++ concurrent_flat_map& xref:#concurrent_flat_map_initializer_list_assignment[operator++=++](std::initializer_list); allocator_type xref:#concurrent_flat_map_get_allocator[get_allocator]() const noexcept; @@ -316,8 +317,7 @@ https://en.cppreference.com/w/cpp/named_req/Erasable[Erasable^] from the table. |_Allocator_ |An allocator whose value type is the same as the table's value type. -`std::allocator_traits::pointer` and `std::allocator_traits::const_pointer` -must be convertible to/from `value_type*` and `const value_type*`, respectively. +Allocators using https://en.cppreference.com/w/cpp/named_req/Allocator#Fancy_pointers[fancy pointers] are supported. |=== @@ -673,8 +673,9 @@ Concurrency:;; Blocking on `*this` and `other`. ==== Move Assignment ```c++ concurrent_flat_map& operator=(concurrent_flat_map&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + noexcept((boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value); ``` The move assignment operator. Destroys previously existing elements, swaps the hash function and predicate from `other`, and move-assigns the allocator from `other` if `Alloc::propagate_on_container_move_assignment` exists and `Alloc::propagate_on_container_move_assignment::value` is `true`. diff --git a/doc/unordered/unordered_flat_map.adoc b/doc/unordered/unordered_flat_map.adoc index 7018433b..7518ba09 100644 --- a/doc/unordered/unordered_flat_map.adoc +++ b/doc/unordered/unordered_flat_map.adoc @@ -97,9 +97,10 @@ namespace boost { const allocator_type& a); xref:#unordered_flat_map_destructor[~unordered_flat_map](); unordered_flat_map& xref:#unordered_flat_map_copy_assignment[operator++=++](const unordered_flat_map& other); - unordered_flat_map& xref:#unordered_flat_map_move_assignment[operator++=++](unordered_flat_map&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + unordered_flat_map& xref:#unordered_flat_map_move_assignment[operator++=++](unordered_flat_map&& other) ++noexcept( + (boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value);++ unordered_flat_map& xref:#unordered_flat_map_initializer_list_assignment[operator++=++](std::initializer_list); allocator_type xref:#unordered_flat_map_get_allocator[get_allocator]() const noexcept; @@ -312,8 +313,7 @@ https://en.cppreference.com/w/cpp/named_req/Erasable[Erasable^] from the contain |_Allocator_ |An allocator whose value type is the same as the container's value type. -`std::allocator_traits::pointer` and `std::allocator_traits::const_pointer` -must be convertible to/from `value_type*` and `const value_type*`, respectively. +Allocators using https://en.cppreference.com/w/cpp/named_req/Allocator#Fancy_pointers[fancy pointers] are supported. |=== @@ -632,8 +632,9 @@ Requires:;; `value_type` is https://en.cppreference.com/w/cpp/named_req/CopyInse ==== Move Assignment ```c++ unordered_flat_map& operator=(unordered_flat_map&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + noexcept((boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value); ``` The move assignment operator. Destroys previously existing elements, swaps the hash function and predicate from `other`, and move-assigns the allocator from `other` if `Alloc::propagate_on_container_move_assignment` exists and `Alloc::propagate_on_container_move_assignment::value` is `true`. @@ -1325,7 +1326,7 @@ void rehash(size_type n); Changes if necessary the size of the bucket array so that there are at least `n` buckets, and so that the load factor is less than or equal to the maximum load factor. When applicable, this will either grow or shrink the `bucket_count()` associated with the container. -When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. +When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. If the provided Allocator uses fancy pointers, a default allocation is subsequently performed. Invalidates iterators, pointers and references, and changes the order of elements. diff --git a/doc/unordered/unordered_flat_set.adoc b/doc/unordered/unordered_flat_set.adoc index 68410ba5..9fbad161 100644 --- a/doc/unordered/unordered_flat_set.adoc +++ b/doc/unordered/unordered_flat_set.adoc @@ -91,9 +91,10 @@ namespace boost { const allocator_type& a); xref:#unordered_flat_set_destructor[~unordered_flat_set](); unordered_flat_set& xref:#unordered_flat_set_copy_assignment[operator++=++](const unordered_flat_set& other); - unordered_flat_set& xref:#unordered_flat_set_move_assignment[operator++=++](unordered_flat_set&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + unordered_flat_set& xref:#unordered_flat_set_move_assignment[operator++=++](unordered_flat_set&& other) ++noexcept( + (boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value);++ unordered_flat_set& xref:#unordered_flat_set_initializer_list_assignment[operator++=++](std::initializer_list); allocator_type xref:#unordered_flat_set_get_allocator[get_allocator]() const noexcept; @@ -261,8 +262,7 @@ and https://en.cppreference.com/w/cpp/named_req/Erasable[Erasable^] from the con |_Allocator_ |An allocator whose value type is the same as the container's value type. -`std::allocator_traits::pointer` and `std::allocator_traits::const_pointer` -must be convertible to/from `value_type*` and `const value_type*`, respectively. +Allocators using https://en.cppreference.com/w/cpp/named_req/Allocator#Fancy_pointers[fancy pointers] are supported. |=== @@ -565,8 +565,9 @@ Requires:;; `value_type` is https://en.cppreference.com/w/cpp/named_req/CopyInse ==== Move Assignment ```c++ unordered_flat_set& operator=(unordered_flat_set&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + noexcept((boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value); ``` The move assignment operator. Destroys previously existing elements, swaps the hash function and predicate from `other`, and move-assigns the allocator from `other` if `Alloc::propagate_on_container_move_assignment` exists and `Alloc::propagate_on_container_move_assignment::value` is `true`. @@ -1087,7 +1088,7 @@ void rehash(size_type n); Changes if necessary the size of the bucket array so that there are at least `n` buckets, and so that the load factor is less than or equal to the maximum load factor. When applicable, this will either grow or shrink the `bucket_count()` associated with the container. -When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. +When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. If the provided Allocator uses fancy pointers, a default allocation is subsequently performed. Invalidates iterators, pointers and references, and changes the order of elements. diff --git a/doc/unordered/unordered_node_map.adoc b/doc/unordered/unordered_node_map.adoc index fcc46d5c..8e973e5a 100644 --- a/doc/unordered/unordered_node_map.adoc +++ b/doc/unordered/unordered_node_map.adoc @@ -95,9 +95,10 @@ namespace boost { const allocator_type& a); xref:#unordered_node_map_destructor[~unordered_node_map](); unordered_node_map& xref:#unordered_node_map_copy_assignment[operator++=++](const unordered_node_map& other); - unordered_node_map& xref:#unordered_node_map_move_assignment[operator++=++](unordered_node_map&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + unordered_node_map& xref:#unordered_node_map_move_assignment[operator++=++](unordered_node_map&& other) ++noexcept( + (boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value);++ unordered_node_map& xref:#unordered_node_map_initializer_list_assignment[operator++=++](std::initializer_list); allocator_type xref:#unordered_node_map_get_allocator[get_allocator]() const noexcept; @@ -314,8 +315,7 @@ https://en.cppreference.com/w/cpp/named_req/Erasable[Erasable^] from the contain |_Allocator_ |An allocator whose value type is the same as the container's value type. -`std::allocator_traits::pointer` and `std::allocator_traits::const_pointer` -must be convertible to/from `value_type*` and `const value_type*`, respectively. +Allocators using https://en.cppreference.com/w/cpp/named_req/Allocator#Fancy_pointers[fancy pointers] are supported. |=== @@ -649,8 +649,9 @@ Requires:;; `value_type` is https://en.cppreference.com/w/cpp/named_req/CopyInse ==== Move Assignment ```c++ unordered_node_map& operator=(unordered_node_map&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + noexcept((boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value); ``` The move assignment operator. Destroys previously existing elements, swaps the hash function and predicate from `other`, and move-assigns the allocator from `other` if `Alloc::propagate_on_container_move_assignment` exists and `Alloc::propagate_on_container_move_assignment::value` is `true`. @@ -1406,7 +1407,7 @@ void rehash(size_type n); Changes if necessary the size of the bucket array so that there are at least `n` buckets, and so that the load factor is less than or equal to the maximum load factor. When applicable, this will either grow or shrink the `bucket_count()` associated with the container. -When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. +When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. If the provided Allocator uses fancy pointers, a default allocation is subsequently performed. Invalidates iterators and changes the order of elements. diff --git a/doc/unordered/unordered_node_set.adoc b/doc/unordered/unordered_node_set.adoc index 44a7fc9e..cbf00444 100644 --- a/doc/unordered/unordered_node_set.adoc +++ b/doc/unordered/unordered_node_set.adoc @@ -90,9 +90,10 @@ namespace boost { const allocator_type& a); xref:#unordered_node_set_destructor[~unordered_node_set](); unordered_node_set& xref:#unordered_node_set_copy_assignment[operator++=++](const unordered_node_set& other); - unordered_node_set& xref:#unordered_node_set_move_assignment[operator++=++](unordered_node_set&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + unordered_node_set& xref:#unordered_node_set_move_assignment[operator++=++](unordered_node_set&& other) ++noexcept( + (boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value);++ unordered_node_set& xref:#unordered_node_set_initializer_list_assignment[operator++=++](std::initializer_list); allocator_type xref:#unordered_node_set_get_allocator[get_allocator]() const noexcept; @@ -264,8 +265,7 @@ namespace boost { |_Allocator_ |An allocator whose value type is the same as the container's value type. -`std::allocator_traits::pointer` and `std::allocator_traits::const_pointer` -must be convertible to/from `value_type*` and `const value_type*`, respectively. +Allocators using https://en.cppreference.com/w/cpp/named_req/Allocator#Fancy_pointers[fancy pointers] are supported. |=== @@ -602,8 +602,9 @@ Requires:;; `value_type` is https://en.cppreference.com/w/cpp/named_req/CopyInse ==== Move Assignment ```c++ unordered_node_set& operator=(unordered_node_set&& other) - noexcept(boost::allocator_traits::is_always_equal::value || - boost::allocator_traits::propagate_on_container_move_assignment::value); + noexcept((boost::allocator_traits::is_always_equal::value || + boost::allocator_traits::propagate_on_container_move_assignment::value) && + std::is_same::value); ``` The move assignment operator. Destroys previously existing elements, swaps the hash function and predicate from `other`, and move-assigns the allocator from `other` if `Alloc::propagate_on_container_move_assignment` exists and `Alloc::propagate_on_container_move_assignment::value` is `true`. @@ -1188,7 +1189,7 @@ void rehash(size_type n); Changes if necessary the size of the bucket array so that there are at least `n` buckets, and so that the load factor is less than or equal to the maximum load factor. When applicable, this will either grow or shrink the `bucket_count()` associated with the container. -When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. +When `size() == 0`, `rehash(0)` will deallocate the underlying buckets array. If the provided Allocator uses fancy pointers, a default allocation is subsequently performed. Invalidates iterators and changes the order of elements. diff --git a/include/boost/unordered/concurrent_flat_map.hpp b/include/boost/unordered/concurrent_flat_map.hpp index e04f84b6..c95a3015 100644 --- a/include/boost/unordered/concurrent_flat_map.hpp +++ b/include/boost/unordered/concurrent_flat_map.hpp @@ -92,7 +92,10 @@ namespace boost { using type_policy = detail::foa::flat_map_types; - detail::foa::concurrent_table table_; + using table_type = + detail::foa::concurrent_table; + + table_type table_; template bool friend operator==(concurrent_flat_map const& lhs, @@ -248,10 +251,8 @@ namespace boost { return *this; } - concurrent_flat_map& operator=(concurrent_flat_map&& rhs) - noexcept(boost::allocator_is_always_equal::type::value || - boost::allocator_propagate_on_container_move_assignment< - Allocator>::type::value) + concurrent_flat_map& operator=(concurrent_flat_map&& rhs) noexcept( + noexcept(std::declval() = std::declval())) { table_ = std::move(rhs.table_); return *this; diff --git a/include/boost/unordered/detail/foa/concurrent_table.hpp b/include/boost/unordered/detail/foa/concurrent_table.hpp index b8335de9..3a90c2f4 100644 --- a/include/boost/unordered/detail/foa/concurrent_table.hpp +++ b/include/boost/unordered/detail/foa/concurrent_table.hpp @@ -247,16 +247,24 @@ group_access* dummy_group_accesses() /* subclasses table_arrays to add an additional group_access array */ -template -struct concurrent_table_arrays:table_arrays +template +struct concurrent_table_arrays:table_arrays { - using super=table_arrays; + using group_access_allocator_type= + typename boost::allocator_rebind::type; + using group_access_pointer= + typename boost::allocator_pointer::type; - concurrent_table_arrays(const super& arrays,group_access *pga): - super{arrays},group_accesses{pga}{} + using super=table_arrays; - template - static concurrent_table_arrays new_(Allocator& al,std::size_t n) + concurrent_table_arrays(const super& arrays,group_access_pointer pga): + super{arrays},group_accesses_{pga}{} + + group_access* group_accesses()const noexcept{ + return boost::to_address(group_accesses_); + } + + static concurrent_table_arrays new_(group_access_allocator_type al,std::size_t n) { super x{super::new_(al,n)}; BOOST_TRY{ @@ -269,54 +277,61 @@ struct concurrent_table_arrays:table_arrays BOOST_CATCH_END } - template - static concurrent_table_arrays new_group_access(Allocator& al,const super& x) + static void set_group_access( + group_access_allocator_type al,concurrent_table_arrays& arrays) { - concurrent_table_arrays arrays{x,nullptr}; - if(!arrays.elements){ - arrays.group_accesses=dummy_group_accesses(); - } - else{ - using access_alloc= - typename boost::allocator_rebind::type; - using access_traits=boost::allocator_traits; + set_group_access( + al,arrays,std::is_same{}); + } - auto aal=access_alloc(al); - arrays.group_accesses=boost::to_address( - access_traits::allocate(aal,arrays.groups_size_mask+1)); + static void set_group_access( + group_access_allocator_type al, + concurrent_table_arrays& arrays, + std::false_type /* fancy pointers */) + { + arrays.group_accesses_= + boost::allocator_allocate(al,arrays.groups_size_mask+1); for(std::size_t i=0;i(); + } else { + set_group_access(al,arrays,std::false_type{}); } + } + + static concurrent_table_arrays new_group_access(group_access_allocator_type al,const super& x) + { + concurrent_table_arrays arrays{x,nullptr}; + set_group_access(al,arrays); return arrays; } - template - static void delete_(Allocator& al,concurrent_table_arrays& arrays)noexcept + static void delete_(group_access_allocator_type al,concurrent_table_arrays& arrays)noexcept { delete_group_access(al,arrays); super::delete_(al,arrays); } - template - static void delete_group_access(Allocator& al,concurrent_table_arrays& arrays)noexcept + static void delete_group_access(group_access_allocator_type al,concurrent_table_arrays& arrays)noexcept { - if(arrays.elements){ - using access_alloc= - typename boost::allocator_rebind::type; - using access_traits=boost::allocator_traits; - using pointer=typename access_traits::pointer; - using pointer_traits=boost::pointer_traits; - - auto aal=access_alloc(al); - access_traits::deallocate( - aal,pointer_traits::pointer_to(*arrays.group_accesses), - arrays.groups_size_mask+1); + if(arrays.elements()){ + boost::allocator_deallocate( + al,arrays.group_accesses_,arrays.groups_size_mask+1); } } - group_access *group_accesses; + group_access_pointer group_accesses_; }; struct atomic_size_control @@ -473,20 +488,32 @@ public: concurrent_table(concurrent_table&& x,const Allocator& al_): concurrent_table(std::move(x),al_,x.exclusive_access()){} - concurrent_table(compatible_nonconcurrent_table&& x): + template + concurrent_table( + compatible_nonconcurrent_table&& x, + arrays_holder&& ah): super{ std::move(x.h()),std::move(x.pred()),std::move(x.al()), - arrays_type(arrays_type::new_group_access( - x.al(), - typename arrays_type::super{ + [&x]{return arrays_type::new_group_access( + x.al(),typename arrays_type::super{ x.arrays.groups_size_index,x.arrays.groups_size_mask, - reinterpret_cast(x.arrays.groups), - reinterpret_cast(x.arrays.elements)})), + to_pointer( + reinterpret_cast(x.arrays.groups())), + x.arrays.elements_});}, size_ctrl_type{x.size_ctrl.ml,x.size_ctrl.size}} { - x.empty_initialize(); + ah.release(); + x.arrays=ah.get(); + x.size_ctrl.ml=x.initial_max_load(); + x.size_ctrl.size=0; } + concurrent_table(compatible_nonconcurrent_table&& x): + concurrent_table(std::move(x), arrays_holder< + typename compatible_nonconcurrent_table::arrays_type,Allocator + >{compatible_nonconcurrent_table::arrays_type::new_(x.al(),0),x.al()}) + {} + ~concurrent_table()=default; concurrent_table& operator=(const concurrent_table& x) @@ -496,7 +523,8 @@ public: return *this; } - concurrent_table& operator=(concurrent_table&& x) + concurrent_table& operator=(concurrent_table&& x)noexcept( + noexcept(std::declval() = std::declval())) { auto lck=exclusive_access(*this,x); super::operator=(std::move(x)); @@ -967,18 +995,18 @@ private: inline group_shared_lock_guard access(group_shared,std::size_t pos)const { - return this->arrays.group_accesses[pos].shared_access(); + return this->arrays.group_accesses()[pos].shared_access(); } inline group_exclusive_lock_guard access( group_exclusive,std::size_t pos)const { - return this->arrays.group_accesses[pos].exclusive_access(); + return this->arrays.group_accesses()[pos].exclusive_access(); } inline group_insert_counter_type& insert_counter(std::size_t pos)const { - return this->arrays.group_accesses[pos].insert_counter(); + return this->arrays.group_accesses()[pos].insert_counter(); } /* Const casts value_type& according to the level of group access for @@ -1097,10 +1125,10 @@ private: prober pb(pos0); do{ auto pos=pb.get(); - auto pg=this->arrays.groups+pos; + auto pg=this->arrays.groups()+pos; auto mask=pg->match(hash); if(mask){ - auto p=this->arrays.elements+pos*N; + auto p=this->arrays.elements()+pos*N; BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N); auto lck=access(access_mode,pos); do{ @@ -1313,7 +1341,7 @@ private: if(BOOST_LIKELY(rsize.succeeded())){ for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){ auto pos=pb.get(); - auto pg=this->arrays.groups+pos; + auto pg=this->arrays.groups()+pos; auto lck=access(group_exclusive{},pos); auto mask=pg->match_available(); if(BOOST_LIKELY(mask!=0)){ @@ -1323,7 +1351,7 @@ private: /* other thread inserted from pos0, need to start over */ goto startover; } - auto p=this->arrays.elements+pos*N+n; + auto p=this->arrays.elements()+pos*N+n; this->construct_element(p,std::forward(args)...); rslot.commit(); rsize.commit(); @@ -1373,11 +1401,11 @@ private: auto for_all_elements_while(GroupAccessMode access_mode,F f)const ->decltype(f(nullptr,0,nullptr),bool()) { - auto p=this->arrays.elements; + auto p=this->arrays.elements(); if(p){ - for(auto pg=this->arrays.groups,last=pg+this->arrays.groups_size_mask+1; + for(auto pg=this->arrays.groups(),last=pg+this->arrays.groups_size_mask+1; pg!=last;++pg,p+=N){ - auto lck=access(access_mode,(std::size_t)(pg-this->arrays.groups)); + auto lck=access(access_mode,(std::size_t)(pg-this->arrays.groups())); auto mask=this->match_really_occupied(pg,last); while(mask){ auto n=unchecked_countr_zero(mask); @@ -1405,13 +1433,13 @@ private: GroupAccessMode access_mode,ExecutionPolicy&& policy,F f)const ->decltype(f(nullptr,0,nullptr),void()) { - if(!this->arrays.elements)return; - auto first=this->arrays.groups, + if(!this->arrays.elements())return; + auto first=this->arrays.groups(), last=first+this->arrays.groups_size_mask+1; std::for_each(std::forward(policy),first,last, [&,this](group_type& g){ auto pos=static_cast(&g-first); - auto p=this->arrays.elements+pos*N; + auto p=this->arrays.elements()+pos*N; auto lck=access(access_mode,pos); auto mask=this->match_really_occupied(&g,last); while(mask){ @@ -1427,13 +1455,13 @@ private: bool for_all_elements_while( GroupAccessMode access_mode,ExecutionPolicy&& policy,F f)const { - if(!this->arrays.elements)return true; - auto first=this->arrays.groups, + if(!this->arrays.elements())return true; + auto first=this->arrays.groups(), last=first+this->arrays.groups_size_mask+1; return std::all_of(std::forward(policy),first,last, [&,this](group_type& g){ auto pos=static_cast(&g-first); - auto p=this->arrays.elements+pos*N; + auto p=this->arrays.elements()+pos*N; auto lck=access(access_mode,pos); auto mask=this->match_really_occupied(&g,last); while(mask){ diff --git a/include/boost/unordered/detail/foa/core.hpp b/include/boost/unordered/detail/foa/core.hpp index 2f3800db..c1fb4850 100644 --- a/include/boost/unordered/detail/foa/core.hpp +++ b/include/boost/unordered/detail/foa/core.hpp @@ -938,74 +938,145 @@ Group* dummy_groups() const_cast(storage)); } -template +template< + typename Ptr,typename Ptr2, + typename std::enable_if::value>::type* = nullptr +> +Ptr to_pointer(Ptr2 p) +{ + if(!p){return nullptr;} + return boost::pointer_traits::pointer_to(*p); +} + +template +Ptr to_pointer(Ptr p) +{ + return p; +} + +template +struct arrays_holder +{ + arrays_holder(Arrays const& arrays, Allocator const& al) + :arrays_{arrays},al_{al} + {} + + arrays_holder(arrays_holder const&)=delete; + arrays_holder& operator=(arrays_holder const&)=delete; + + ~arrays_holder() + { + if (!released_){ + arrays_.delete_(al_,arrays_); + } + } + + Arrays const& get()const + { + return arrays_; + } + + void release() + { + released_=true; + } + +private: + Arrays arrays_; + Allocator al_; + bool released_=false; +}; + +template struct table_arrays { + using allocator_type=typename boost::allocator_rebind::type; + using value_type=Value; using group_type=Group; static constexpr auto N=group_type::N; using size_policy=SizePolicy; + using value_type_pointer= + typename boost::allocator_pointer::type; + using group_type_pointer= + typename boost::pointer_traits::template + rebind; + using group_type_pointer_traits=boost::pointer_traits; - table_arrays(std::size_t gsi,std::size_t gsm,group_type *pg,value_type *pe): - groups_size_index{gsi},groups_size_mask{gsm},groups{pg},elements{pe}{} + table_arrays(std::size_t gsi,std::size_t gsm,group_type_pointer pg,value_type_pointer pe): + groups_size_index{gsi},groups_size_mask{gsm},groups_{pg},elements_{pe}{} - template - static table_arrays new_(Allocator& al,std::size_t n) + value_type* elements()const noexcept{return boost::to_address(elements_);} + group_type* groups()const noexcept{return boost::to_address(groups_);} + + static void set_arrays(table_arrays& arrays,allocator_type al,std::size_t n) { - using storage_allocator= - typename boost::allocator_rebind::type; - using storage_traits=boost::allocator_traits; + return set_arrays( + arrays,al,n,std::is_same{}); + } + static void set_arrays( + table_arrays& arrays,allocator_type al,std::size_t,std::false_type /* always allocate */) + { + using storage_traits=boost::allocator_traits; + auto groups_size_index=arrays.groups_size_index; + auto groups_size=size_policy::size(groups_size_index); + + auto sal=allocator_type(al); + arrays.elements_=storage_traits::allocate(sal,buffer_size(groups_size)); + + /* Align arrays.groups to sizeof(group_type). table_iterator critically + * depends on such alignment for its increment operation. + */ + + auto p=reinterpret_cast(arrays.elements()+groups_size*N-1); + p+=(uintptr_t(sizeof(group_type))- + reinterpret_cast(p))%sizeof(group_type); + arrays.groups_=group_type_pointer_traits::pointer_to(*reinterpret_cast(p)); + + initialize_groups( + arrays.groups(),groups_size, + std::integral_constant< + bool, +#if BOOST_WORKAROUND(BOOST_LIBSTDCXX_VERSION,<50000) + /* std::is_trivially_constructible not provided */ + boost::has_trivial_constructor::value +#else + std::is_trivially_constructible::value +#endif + >{}); + arrays.groups()[groups_size-1].set_sentinel(); + } + + static void set_arrays( + table_arrays& arrays,allocator_type al,std::size_t n,std::true_type /* optimize for n==0*/) + { + if(!n){ + arrays.groups_=dummy_groups(); + } + else{ + set_arrays(arrays,al,n,std::false_type{}); + } + } + + static table_arrays new_(allocator_type al,std::size_t n) + { auto groups_size_index=size_index_for(n); auto groups_size=size_policy::size(groups_size_index); table_arrays arrays{groups_size_index,groups_size-1,nullptr,nullptr}; - if(!n){ - arrays.groups=dummy_groups(); - } - else{ - auto sal=storage_allocator(al); - arrays.elements=boost::to_address( - storage_traits::allocate(sal,buffer_size(groups_size))); - - /* Align arrays.groups to sizeof(group_type). table_iterator critically - * depends on such alignment for its increment operation. - */ - - auto p=reinterpret_cast(arrays.elements+groups_size*N-1); - p+=(uintptr_t(sizeof(group_type))- - reinterpret_cast(p))%sizeof(group_type); - arrays.groups=reinterpret_cast(p); - - initialize_groups( - arrays.groups,groups_size, - std::integral_constant< - bool, -#if BOOST_WORKAROUND(BOOST_LIBSTDCXX_VERSION,<50000) - /* std::is_trivially_constructible not provided */ - boost::has_trivial_constructor::value -#else - std::is_trivially_constructible::value -#endif - >{}); - arrays.groups[groups_size-1].set_sentinel(); - } + set_arrays(arrays,al,n); return arrays; } - template - static void delete_(Allocator& al,table_arrays& arrays)noexcept + static void delete_(allocator_type al,table_arrays& arrays)noexcept { - using storage_alloc=typename boost::allocator_rebind::type; - using storage_traits=boost::allocator_traits; - using pointer=typename storage_traits::pointer; - using pointer_traits=boost::pointer_traits; + using storage_traits=boost::allocator_traits; - auto sal=storage_alloc(al); - if(arrays.elements){ + auto sal=allocator_type(al); + if(arrays.elements()){ storage_traits::deallocate( - sal,pointer_traits::pointer_to(*arrays.elements), - buffer_size(arrays.groups_size_mask+1)); + sal,arrays.elements_,buffer_size(arrays.groups_size_mask+1)); } } @@ -1024,7 +1095,7 @@ struct table_arrays } static void initialize_groups( - group_type* groups_,std::size_t size,std::true_type /* memset */) + group_type* pg,std::size_t size,std::true_type /* memset */) { /* memset faster/not slower than manual, assumes all zeros is group_type's * default layout. @@ -1033,19 +1104,19 @@ struct table_arrays */ std::memset( - reinterpret_cast(groups_),0,sizeof(group_type)*size); + reinterpret_cast(pg),0,sizeof(group_type)*size); } static void initialize_groups( - group_type* groups_,std::size_t size,std::false_type /* manual */) + group_type* pg,std::size_t size,std::false_type /* manual */) { - while(size--!=0)::new (groups_++) group_type(); + while(size--!=0)::new (pg++) group_type(); } - std::size_t groups_size_index; - std::size_t groups_size_mask; - group_type *groups; - value_type *elements; + std::size_t groups_size_index; + std::size_t groups_size_mask; + group_type_pointer groups_; + value_type_pointer elements_; }; struct if_constexpr_void_else{void operator()()const{}}; @@ -1258,8 +1329,12 @@ public: >::type; using alloc_traits=boost::allocator_traits; using element_type=typename type_policy::element_type; - using arrays_type=Arrays; + using arrays_type=Arrays; using size_ctrl_type=SizeControl; + static constexpr auto uses_fancy_pointers=!std::is_same< + typename alloc_traits::pointer, + typename alloc_traits::value_type* + >::value; using key_type=typename type_policy::key_type; using init_type=typename type_policy::init_type; @@ -1283,31 +1358,44 @@ public: size_ctrl{initial_max_load(),0} {} - /* bare transfer ctor for concurrent/non-concurrent interop */ - + /* genericize on an ArraysFn so that we can do things like delay an + * allocation for the group_access data required by cfoa after the move + * constructors of Hash, Pred have been invoked + */ + template table_core( Hash&& h_,Pred&& pred_,Allocator&& al_, - const arrays_type& arrays_,const size_ctrl_type& size_ctrl_): + ArraysFn arrays_fn,const size_ctrl_type& size_ctrl_): hash_base{empty_init,std::move(h_)}, pred_base{empty_init,std::move(pred_)}, allocator_base{empty_init,std::move(al_)}, - arrays(arrays_),size_ctrl(size_ctrl_) + arrays(arrays_fn()),size_ctrl(size_ctrl_) {} table_core(const table_core& x): table_core{x,alloc_traits::select_on_container_copy_construction(x.al())}{} + template + table_core(table_core&& x,arrays_holder&& ah,ArraysFn arrays_fn): + table_core( + std::move(x.h()),std::move(x.pred()),std::move(x.al()),arrays_fn,x.size_ctrl) + { + ah.release(); + x.arrays=ah.get(); + x.size_ctrl.ml=x.initial_max_load(); + x.size_ctrl.size=0; + } + table_core(table_core&& x) noexcept( std::is_nothrow_move_constructible::value&& std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value): + std::is_nothrow_move_constructible::value&& + !uses_fancy_pointers): table_core{ - std::move(x.h()),std::move(x.pred()),std::move(x.al()), - x.arrays,x.size_ctrl} - { - x.empty_initialize(); - } + std::move(x),arrays_holder{x.new_arrays(0),x.al()}, + [&x]{return x.arrays;}} + {} table_core(const table_core& x,const Allocator& al_): table_core{std::size_t(std::ceil(float(x.size())/mlf)),x.h(),x.pred(),al_} @@ -1345,11 +1433,17 @@ public: delete_arrays(arrays); } - void empty_initialize()noexcept + std::size_t initial_max_load()const { - arrays=new_arrays(0); - size_ctrl.ml=initial_max_load(); - size_ctrl.size=0; + static constexpr std::size_t small_capacity=2*N-1; + + auto capacity_=capacity(); + if(capacity_<=small_capacity){ + return capacity_; /* we allow 100% usage */ + } + else{ + return (std::size_t)(mlf*(float)(capacity_)); + } } table_core& operator=(const table_core& x) @@ -1397,8 +1491,8 @@ public: table_core& operator=(table_core&& x) noexcept( - alloc_traits::propagate_on_container_move_assignment::value|| - alloc_traits::is_always_equal::value) + (alloc_traits::propagate_on_container_move_assignment::value|| + alloc_traits::is_always_equal::value)&&!uses_fancy_pointers) { BOOST_UNORDERED_STATIC_ASSERT_HASH_PRED(Hash, Pred) @@ -1489,11 +1583,12 @@ public: prober pb(pos0); do{ auto pos=pb.get(); - auto pg=arrays.groups+pos; + auto pg=arrays.groups()+pos; auto mask=pg->match(hash); if(mask){ - BOOST_UNORDERED_ASSUME(arrays.elements!=nullptr); - auto p=arrays.elements+pos*N; + auto elements=arrays.elements(); + BOOST_UNORDERED_ASSUME(elements!=nullptr); + auto p=elements+pos*N; BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N); do{ auto n=unchecked_countr_zero(mask); @@ -1542,9 +1637,9 @@ public: void clear()noexcept { - auto p=arrays.elements; + auto p=arrays.elements(); if(p){ - for(auto pg=arrays.groups,last=pg+arrays.groups_size_mask+1; + for(auto pg=arrays.groups(),last=pg+arrays.groups_size_mask+1; pg!=last;++pg,p+=N){ auto mask=match_really_occupied(pg,last); while(mask){ @@ -1554,7 +1649,7 @@ public: /* we wipe the entire metadata to reset the overflow byte as well */ pg->initialize(); } - arrays.groups[arrays.groups_size_mask].set_sentinel(); + arrays.groups()[arrays.groups_size_mask].set_sentinel(); size_ctrl.ml=initial_max_load(); size_ctrl.size=0; } @@ -1565,7 +1660,7 @@ public: std::size_t capacity()const noexcept { - return arrays.elements?(arrays.groups_size_mask+1)*N-1:0; + return arrays.elements()?(arrays.groups_size_mask+1)*N-1:0; } float load_factor()const noexcept @@ -1784,9 +1879,9 @@ public: static auto for_all_elements_while(const arrays_type& arrays_,F f) ->decltype(f(nullptr,0,nullptr),bool()) { - auto p=arrays_.elements; + auto p=arrays_.elements(); if(p){ - for(auto pg=arrays_.groups,last=pg+arrays_.groups_size_mask+1; + for(auto pg=arrays_.groups(),last=pg+arrays_.groups_size_mask+1; pg!=last;++pg,p+=N){ auto mask=match_really_occupied(pg,last); while(mask){ @@ -1887,7 +1982,7 @@ private: void fast_copy_elements_from(const table_core& x) { - if(arrays.elements&&x.arrays.elements){ + if(arrays.elements()&&x.arrays.elements()){ copy_elements_array_from(x); copy_groups_array_from(x); size_ctrl.ml=std::size_t(x.size_ctrl.ml); @@ -1921,8 +2016,8 @@ private: * copy-assignable when we're relying on trivial copy constructibility. */ std::memcpy( - reinterpret_cast(arrays.elements), - reinterpret_cast(x.arrays.elements), + reinterpret_cast(arrays.elements()), + reinterpret_cast(x.arrays.elements()), x.capacity()*sizeof(value_type)); } @@ -1932,14 +2027,14 @@ private: std::size_t num_constructed=0; BOOST_TRY{ x.for_all_elements([&,this](const element_type* p){ - construct_element(arrays.elements+(p-x.arrays.elements),*p); + construct_element(arrays.elements()+(p-x.arrays.elements()),*p); ++num_constructed; }); } BOOST_CATCH(...){ if(num_constructed){ x.for_all_elements_while([&,this](const element_type* p){ - destroy_element(arrays.elements+(p-x.arrays.elements)); + destroy_element(arrays.elements()+(p-x.arrays.elements())); return --num_constructed!=0; }); } @@ -1964,15 +2059,17 @@ private: const table_core& x, std::true_type /* -> memcpy */) { std::memcpy( - arrays.groups,x.arrays.groups, + arrays.groups(),x.arrays.groups(), (arrays.groups_size_mask+1)*sizeof(group_type)); } void copy_groups_array_from( const table_core& x, std::false_type /* -> manual */) { + auto pg=arrays.groups(); + auto xpg=x.arrays.groups(); for(std::size_t i=0;i(pg)+pos); } - std::size_t initial_max_load()const - { - static constexpr std::size_t small_capacity=2*N-1; - - auto capacity_=capacity(); - if(capacity_<=small_capacity){ - return capacity_; /* we allow 100% usage */ - } - else{ - return (std::size_t)(mlf*(float)(capacity_)); - } - } - static std::size_t capacity_for(std::size_t n) { return size_policy::size(size_index_for(n))*N-1; @@ -2102,11 +2186,11 @@ private: { for(prober pb(pos0);;pb.next(arrays_.groups_size_mask)){ auto pos=pb.get(); - auto pg=arrays_.groups+pos; + auto pg=arrays_.groups()+pos; auto mask=pg->match_available(); if(BOOST_LIKELY(mask!=0)){ auto n=unchecked_countr_zero(mask); - auto p=arrays_.elements+pos*N+n; + auto p=arrays_.elements()+pos*N+n; construct_element(p,std::forward(args)...); pg->set(n,hash); return {pg,n,p}; diff --git a/include/boost/unordered/detail/foa/element_type.hpp b/include/boost/unordered/detail/foa/element_type.hpp index 650e3b4f..fd226638 100644 --- a/include/boost/unordered/detail/foa/element_type.hpp +++ b/include/boost/unordered/detail/foa/element_type.hpp @@ -9,26 +9,30 @@ #ifndef BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP #define BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP +#include + namespace boost{ namespace unordered{ namespace detail{ namespace foa{ -template +template struct element_type { using value_type=T; - value_type* p; + using pointer=typename boost::pointer_traits::template rebind; + + pointer p; /* * we use a deleted copy constructor here so the type is no longer * trivially copy-constructible which inhibits our memcpy * optimizations when copying the tables */ - element_type() = default; - element_type(value_type* p_):p(p_){} - element_type(element_type const&) = delete; - element_type(element_type&& rhs) noexcept + element_type()=default; + element_type(pointer p_):p(p_){} + element_type(element_type const&)=delete; + element_type(element_type&& rhs)noexcept { p = rhs.p; rhs.p = nullptr; diff --git a/include/boost/unordered/detail/foa/node_map_types.hpp b/include/boost/unordered/detail/foa/node_map_types.hpp index 8b4a4b91..473d97d7 100644 --- a/include/boost/unordered/detail/foa/node_map_types.hpp +++ b/include/boost/unordered/detail/foa/node_map_types.hpp @@ -13,7 +13,7 @@ namespace boost { namespace unordered { namespace detail { namespace foa { - template struct node_map_types + template struct node_map_types { using key_type = Key; using mapped_type = T; @@ -24,7 +24,7 @@ namespace boost { using value_type = std::pair; using moved_type = std::pair; - using element_type = foa::element_type; + using element_type = foa::element_type; static value_type& value_from(element_type const& x) { @@ -83,18 +83,15 @@ namespace boost { template static void construct(A& al, element_type* p, Args&&... args) { - p->p = boost::to_address(boost::allocator_allocate(al, 1)); + p->p = boost::allocator_allocate(al, 1); BOOST_TRY { - boost::allocator_construct(al, p->p, std::forward(args)...); + boost::allocator_construct( + al, boost::to_address(p->p), std::forward(args)...); } BOOST_CATCH(...) { - using pointer_type = typename boost::allocator_pointer::type; - using pointer_traits = boost::pointer_traits; - - boost::allocator_deallocate( - al, pointer_traits::pointer_to(*(p->p)), 1); + boost::allocator_deallocate(al, p->p, 1); BOOST_RETHROW } BOOST_CATCH_END @@ -114,12 +111,8 @@ namespace boost { static void destroy(A& al, element_type* p) noexcept { if (p->p) { - using pointer_type = typename boost::allocator_pointer::type; - using pointer_traits = boost::pointer_traits; - - destroy(al, p->p); - boost::allocator_deallocate( - al, pointer_traits::pointer_to(*(p->p)), 1); + destroy(al, boost::to_address(p->p)); + boost::allocator_deallocate(al, p->p, 1); } } }; diff --git a/include/boost/unordered/detail/foa/node_set_types.hpp b/include/boost/unordered/detail/foa/node_set_types.hpp index f8aaa8e0..4d01d2fe 100644 --- a/include/boost/unordered/detail/foa/node_set_types.hpp +++ b/include/boost/unordered/detail/foa/node_set_types.hpp @@ -14,7 +14,7 @@ namespace boost { namespace detail { namespace foa { - template struct node_set_types + template struct node_set_types { using key_type = Key; using init_type = Key; @@ -22,7 +22,7 @@ namespace boost { static Key const& extract(value_type const& key) { return key; } - using element_type = foa::element_type; + using element_type = foa::element_type; static value_type& value_from(element_type const& x) { return *x.p; } static Key const& extract(element_type const& k) { return *k.p; } @@ -53,17 +53,15 @@ namespace boost { template static void construct(A& al, element_type* p, Args&&... args) { - p->p = boost::to_address(boost::allocator_allocate(al, 1)); + p->p = boost::allocator_allocate(al, 1); BOOST_TRY { - boost::allocator_construct(al, p->p, std::forward(args)...); + boost::allocator_construct( + al, boost::to_address(p->p), std::forward(args)...); } BOOST_CATCH(...) { - boost::allocator_deallocate(al, - boost::pointer_traits::type>::pointer_to(*p->p), - 1); + boost::allocator_deallocate(al, p->p, 1); BOOST_RETHROW } BOOST_CATCH_END @@ -78,11 +76,8 @@ namespace boost { static void destroy(A& al, element_type* p) noexcept { if (p->p) { - destroy(al, p->p); - boost::allocator_deallocate(al, - boost::pointer_traits::type>::pointer_to(*(p->p)), - 1); + destroy(al, boost::to_address(p->p)); + boost::allocator_deallocate(al, p->p, 1); } } }; diff --git a/include/boost/unordered/detail/foa/table.hpp b/include/boost/unordered/detail/foa/table.hpp index fb883011..f8308b1e 100644 --- a/include/boost/unordered/detail/foa/table.hpp +++ b/include/boost/unordered/detail/foa/table.hpp @@ -81,12 +81,17 @@ class table; /* internal conversion from const_iterator to iterator */ struct const_iterator_cast_tag{}; -template +template class table_iterator { + using group_pointer_traits=boost::pointer_traits; using type_policy=TypePolicy; using table_element_type=typename type_policy::element_type; - using group_type=Group; + using group_type=typename group_pointer_traits::element_type; + using table_element_pointer= + typename group_pointer_traits::template rebind; + using char_pointer= + typename group_pointer_traits::template rebind; static constexpr auto N=group_type::N; static constexpr auto regular_layout=group_type::regular_layout; @@ -103,22 +108,22 @@ public: table_iterator()=default; template::type* =nullptr> - table_iterator(const table_iterator& x): - pc{x.pc},p{x.p}{} + table_iterator(const table_iterator& x): + pc_{x.pc_},p_{x.p_}{} table_iterator( - const_iterator_cast_tag, const table_iterator& x): - pc{x.pc},p{x.p}{} + const_iterator_cast_tag, const table_iterator& x): + pc_{x.pc_},p_{x.p_}{} inline reference operator*()const noexcept - {return type_policy::value_from(*p);} + {return type_policy::value_from(*p());} inline pointer operator->()const noexcept - {return std::addressof(type_policy::value_from(*p));} + {return std::addressof(type_policy::value_from(*p()));} inline table_iterator& operator++()noexcept{increment();return *this;} inline table_iterator operator++(int)noexcept {auto x=*this;increment();return x;} friend inline bool operator==( const table_iterator& x,const table_iterator& y) - {return x.p==y.p;} + {return x.p()==y.p();} friend inline bool operator!=( const table_iterator& x,const table_iterator& y) {return !(x==y);} @@ -128,81 +133,91 @@ private: template friend class table_erase_return_type; template friend class table; - table_iterator(Group* pg,std::size_t n,const table_element_type* p_): - pc{reinterpret_cast(const_cast(pg))+n}, - p{const_cast(p_)} - {} + table_iterator(group_type* pg,std::size_t n,const table_element_type* p): + pc_{to_pointer( + reinterpret_cast(const_cast(pg))+n)}, + p_{to_pointer(const_cast(p))} + {} + + unsigned char* pc()const noexcept{return boost::to_address(pc_);} + table_element_type* p()const noexcept{return boost::to_address(p_);} inline void increment()noexcept { - BOOST_ASSERT(p!=nullptr); + BOOST_ASSERT(p()!=nullptr); increment(std::integral_constant{}); } inline void increment(std::true_type /* regular layout */)noexcept { + using diff_type= + typename boost::pointer_traits::difference_type; + for(;;){ - ++p; - if(reinterpret_cast(pc)%sizeof(group_type)==N-1){ - pc+=sizeof(group_type)-(N-1); + ++p_; + if(reinterpret_cast(pc())%sizeof(group_type)==N-1){ + pc_+=static_cast(sizeof(group_type)-(N-1)); break; } - ++pc; - if(!group_type::is_occupied(pc))continue; - if(BOOST_UNLIKELY(group_type::is_sentinel(pc)))p=nullptr; + ++pc_; + if(!group_type::is_occupied(pc()))continue; + if(BOOST_UNLIKELY(group_type::is_sentinel(pc())))p_=nullptr; return; } for(;;){ - int mask=reinterpret_cast(pc)->match_occupied(); + int mask=reinterpret_cast(pc())->match_occupied(); if(mask!=0){ auto n=unchecked_countr_zero(mask); - if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ - p=nullptr; + if(BOOST_UNLIKELY(reinterpret_cast(pc())->is_sentinel(n))){ + p_=nullptr; } else{ - pc+=n; - p+=n; + pc_+=static_cast(n); + p_+=static_cast(n); } return; } - pc+=sizeof(group_type); - p+=N; + pc_+=static_cast(sizeof(group_type)); + p_+=static_cast(N); } } inline void increment(std::false_type /* interleaved */)noexcept { - std::size_t n0=reinterpret_cast(pc)%sizeof(group_type); - pc-=n0; + using diff_type= + typename boost::pointer_traits::difference_type; + + std::size_t n0=reinterpret_cast(pc())%sizeof(group_type); + pc_-=static_cast(n0); int mask=( - reinterpret_cast(pc)->match_occupied()>>(n0+1))<<(n0+1); + reinterpret_cast(pc())->match_occupied()>>(n0+1))<<(n0+1); if(!mask){ do{ - pc+=sizeof(group_type); - p+=N; + pc_+=sizeof(group_type); + p_+=N; } - while((mask=reinterpret_cast(pc)->match_occupied())==0); + while((mask=reinterpret_cast(pc())->match_occupied())==0); } auto n=unchecked_countr_zero(mask); - if(BOOST_UNLIKELY(reinterpret_cast(pc)->is_sentinel(n))){ - p=nullptr; + if(BOOST_UNLIKELY(reinterpret_cast(pc())->is_sentinel(n))){ + p_=nullptr; } else{ - pc+=n; - p-=n0; - p+=n; + pc_+=static_cast(n); + p_-=static_cast(n0); + p_+=static_cast(n); } } template friend void serialization_track(Archive& ar,const table_iterator& x) { - if(x.p){ - track_address(ar,x.pc); - track_address(ar,x.p); + if(x.p()){ + track_address(ar,x.pc_); + track_address(ar,x.p_); } } @@ -211,13 +226,13 @@ private: template void serialize(Archive& ar,unsigned int) { - if(!p)pc=nullptr; - serialize_tracked_address(ar,pc); - serialize_tracked_address(ar,p); + if(!p())pc_=nullptr; + serialize_tracked_address(ar,pc_); + serialize_tracked_address(ar,p_); } - unsigned char *pc=nullptr; - table_element_type *p=nullptr; + char_pointer pc_=nullptr; + table_element_pointer p_=nullptr; }; /* Returned by table::erase([const_]iterator) to avoid iterator increment @@ -227,11 +242,11 @@ private: template class table_erase_return_type; -template -class table_erase_return_type> +template +class table_erase_return_type> { - using iterator=table_iterator; - using const_iterator=table_iterator; + using iterator=table_iterator; + using const_iterator=table_iterator; public: /* can't delete it because VS in pre-C++17 mode needs to see it for RVO */ @@ -313,6 +328,9 @@ class table:table_core_impl using locator=typename super::locator; using compatible_concurrent_table= concurrent_table; + using group_type_pointer=typename boost::pointer_traits< + typename boost::allocator_pointer::type + >::template rebind; friend compatible_concurrent_table; public: @@ -324,7 +342,6 @@ public: private: static constexpr bool has_mutable_iterator= !std::is_same::value; - public: using hasher=typename super::hasher; using key_equal=typename super::key_equal; @@ -335,10 +352,10 @@ public: using const_reference=typename super::const_reference; using size_type=typename super::size_type; using difference_type=typename super::difference_type; - using const_iterator=table_iterator; + using const_iterator=table_iterator; using iterator=typename std::conditional< has_mutable_iterator, - table_iterator, + table_iterator, const_iterator>::type; using erase_return_type=table_erase_return_type; @@ -363,9 +380,9 @@ public: iterator begin()noexcept { - iterator it{this->arrays.groups,0,this->arrays.elements}; - if(this->arrays.elements&& - !(this->arrays.groups[0].match_occupied()&0x1))++it; + iterator it{this->arrays.groups(),0,this->arrays.elements()}; + if(this->arrays.elements()&& + !(this->arrays.groups()[0].match_occupied()&0x1))++it; return it; } @@ -431,7 +448,7 @@ public: BOOST_FORCEINLINE erase_return_type erase(const_iterator pos)noexcept { - super::erase(pos.pc,pos.p); + super::erase(pos.pc(),pos.p()); return {pos}; } @@ -462,7 +479,7 @@ public: BOOST_ASSERT(pos!=end()); erase_on_exit e{*this,pos}; (void)e; - return std::move(*pos.p); + return std::move(*pos.p()); } // TODO: should we accept different allocator too? @@ -527,22 +544,32 @@ public: friend bool operator!=(const table& x,const table& y){return !(x==y);} private: - template - table(compatible_concurrent_table&& x,ExclusiveLockGuard): + template + table(compatible_concurrent_table&& x,arrays_holder&& ah): super{ std::move(x.h()),std::move(x.pred()),std::move(x.al()), - arrays_type{ + [&x]{return arrays_type{ x.arrays.groups_size_index,x.arrays.groups_size_mask, - reinterpret_cast(x.arrays.groups), - reinterpret_cast(x.arrays.elements)}, - size_ctrl_type{ - x.size_ctrl.ml,x.size_ctrl.size}} + to_pointer( + reinterpret_cast(x.arrays.groups())), + x.arrays.elements_};}, + size_ctrl_type{x.size_ctrl.ml,x.size_ctrl.size}} { - compatible_concurrent_table::arrays_type::delete_group_access( - this->al(),x.arrays); - x.empty_initialize(); + ah.release(); + + compatible_concurrent_table::arrays_type::delete_group_access(x.al(),x.arrays); + x.arrays=ah.get(); + x.size_ctrl.ml=x.initial_max_load(); + x.size_ctrl.size=0; } + template + table(compatible_concurrent_table&& x,ExclusiveLockGuard): + table(std::move(x),arrays_holder< + typename compatible_concurrent_table::arrays_type,Allocator + >{compatible_concurrent_table::arrays_type::new_(x.al(),0),x.al()}) + {} + struct erase_on_exit { erase_on_exit(table& x_,const_iterator it_):x{x_},it{it_}{} diff --git a/include/boost/unordered/unordered_flat_map.hpp b/include/boost/unordered/unordered_flat_map.hpp index 1e829c1d..b7ea0a82 100644 --- a/include/boost/unordered/unordered_flat_map.hpp +++ b/include/boost/unordered/unordered_flat_map.hpp @@ -141,9 +141,7 @@ namespace boost { } unordered_flat_map(unordered_flat_map&& other) - noexcept(std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value) + noexcept(std::is_nothrow_move_constructible::value) : table_(std::move(other.table_)) { } @@ -697,10 +695,9 @@ namespace boost { return erase_if(map.table_, pred); } - template - void serialize( - Archive & ar, + template + void serialize(Archive& ar, unordered_flat_map& map, unsigned int version) { diff --git a/include/boost/unordered/unordered_flat_set.hpp b/include/boost/unordered/unordered_flat_set.hpp index 01cadf21..2676cafc 100644 --- a/include/boost/unordered/unordered_flat_set.hpp +++ b/include/boost/unordered/unordered_flat_set.hpp @@ -133,9 +133,7 @@ namespace boost { } unordered_flat_set(unordered_flat_set&& other) - noexcept(std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value) + noexcept(std::is_nothrow_move_constructible::value) : table_(std::move(other.table_)) { } @@ -506,10 +504,9 @@ namespace boost { return erase_if(set.table_, pred); } - template - void serialize( - Archive & ar, + template + void serialize(Archive& ar, unordered_flat_set& set, unsigned int version) { diff --git a/include/boost/unordered/unordered_node_map.hpp b/include/boost/unordered/unordered_node_map.hpp index 3329b1b6..0c6a4e4c 100644 --- a/include/boost/unordered/unordered_node_map.hpp +++ b/include/boost/unordered/unordered_node_map.hpp @@ -75,7 +75,8 @@ namespace boost { template class unordered_node_map { - using map_types = detail::foa::node_map_types; + using map_types = detail::foa::node_map_types::type>; using table_type = detail::foa::table::value&& - std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value) + noexcept(std::is_nothrow_move_constructible::value) : table_(std::move(other.table_)) { } @@ -789,10 +788,9 @@ namespace boost { return erase_if(map.table_, pred); } - template - void serialize( - Archive & ar, + template + void serialize(Archive& ar, unordered_node_map& map, unsigned int version) { diff --git a/include/boost/unordered/unordered_node_set.hpp b/include/boost/unordered/unordered_node_set.hpp index 21d3d28f..1dac7405 100644 --- a/include/boost/unordered/unordered_node_set.hpp +++ b/include/boost/unordered/unordered_node_set.hpp @@ -66,7 +66,8 @@ namespace boost { template class unordered_node_set { - using set_types = detail::foa::node_set_types; + using set_types = detail::foa::node_set_types::type>; using table_type = detail::foa::table::value&& - std::is_nothrow_move_constructible::value&& - std::is_nothrow_move_constructible::value) + noexcept(std::is_nothrow_move_constructible::value) : table_(std::move(other.table_)) { } @@ -602,10 +601,9 @@ namespace boost { return erase_if(set.table_, pred); } - template - void serialize( - Archive & ar, + template + void serialize(Archive& ar, unordered_node_set& set, unsigned int version) { diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index d87fcf6d..79883626 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -5,6 +5,7 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) import testing ; +import ../../config/checks/config : requires ; # Adding -Wundef is blocked on (at least) # https://github.com/boostorg/type_traits/issues/165 @@ -15,6 +16,20 @@ local msvc-flags = /wd4494 ; project : requirements + [ requires cxx11_rvalue_references + cxx11_auto_declarations + cxx11_nullptr + cxx11_defaulted_functions + cxx11_final + cxx11_hdr_type_traits + cxx11_hdr_initializer_list + cxx11_static_assert + cxx11_smart_ptr + cxx11_constexpr + cxx11_noexcept + cxx11_decltype + cxx11_alignas + ] pedantic intel:on @@ -33,63 +48,76 @@ project msvc:on ; -run unordered/prime_fmod_tests.cpp ; -run unordered/fwd_set_test.cpp ; -run unordered/fwd_map_test.cpp ; -run unordered/allocator_traits.cpp ; -run unordered/minimal_allocator.cpp ; -run unordered/compile_set.cpp ; -run unordered/compile_map.cpp ; -run unordered/noexcept_tests.cpp ; -run unordered/link_test_1.cpp unordered/link_test_2.cpp ; -run unordered/incomplete_test.cpp ; -run unordered/simple_tests.cpp ; -run unordered/equivalent_keys_tests.cpp ; -run unordered/constructor_tests.cpp ; -run unordered/copy_tests.cpp ; -run unordered/move_tests.cpp ; -run unordered/post_move_tests.cpp ; -run unordered/assign_tests.cpp ; -run unordered/insert_tests.cpp ; -run unordered/insert_stable_tests.cpp ; -run unordered/insert_hint_tests.cpp ; -run unordered/emplace_tests.cpp ; -run unordered/unnecessary_copy_tests.cpp ; -run unordered/erase_tests.cpp : : : BOOST_UNORDERED_SUPPRESS_DEPRECATED ; -run unordered/erase_equiv_tests.cpp ; -run unordered/extract_tests.cpp ; -run unordered/node_handle_tests.cpp ; -run unordered/merge_tests.cpp ; -compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_MAP : insert_node_type_fail_map ; -compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_MULTIMAP : insert_node_type_fail_multimap ; -compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_SET : insert_node_type_fail_set ; -compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_MULTISET : insert_node_type_fail_multiset ; -run unordered/find_tests.cpp ; -run unordered/at_tests.cpp ; -run unordered/bucket_tests.cpp ; -run unordered/load_factor_tests.cpp ; -run unordered/rehash_tests.cpp ; -run unordered/equality_tests.cpp ; -run unordered/swap_tests.cpp ; -run unordered/deduction_tests.cpp ; -run unordered/scoped_allocator.cpp : : : msvc-14.0:no ; -run unordered/transparent_tests.cpp ; -run unordered/reserve_tests.cpp ; -run unordered/contains_tests.cpp ; -run unordered/erase_if.cpp ; -run unordered/scary_tests.cpp ; - -run unordered/compile_set.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_compile_set ; -run unordered/compile_map.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_compile_map ; -run unordered/copy_tests.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_copy ; -run unordered/move_tests.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_move ; -run unordered/assign_tests.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_assign ; - path-constant BOOST_UNORDERED_TEST_DIR : . ; -run unordered/serialization_tests.cpp +run quick.cpp ; + +local FCA_TESTS = + allocator_traits + assign_tests + at_tests + bucket_tests + compile_map + compile_set + constructor_tests + contains_tests + copy_tests + deduction_tests + emplace_tests + equality_tests + equivalent_keys_tests + erase_equiv_tests + erase_if + erase_tests + extract_tests + find_tests + fwd_map_test + fwd_set_test + incomplete_test + insert_hint_tests + insert_stable_tests + insert_tests + load_factor_tests + merge_tests + minimal_allocator + move_tests + narrow_cast_tests + node_handle_tests + noexcept_tests + post_move_tests + prime_fmod_tests + rehash_tests + reserve_tests + scary_tests + scoped_allocator + simple_tests + swap_tests + transparent_tests + unnecessary_copy_tests +; + +for local test in $(FCA_TESTS) +{ + if $(test) = "erase_tests" { + run unordered/$(test).cpp : : : BOOST_UNORDERED_SUPPRESS_DEPRECATED ; + } else if $(test) = "scoped_allocator" { + run unordered/$(test).cpp : : : msvc-14.0:no ; + } else { + run unordered/$(test).cpp ; + } +} + +run unordered/link_test_1.cpp unordered/link_test_2.cpp : : : : link_test ; + +run unordered/compile_set.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_compile_set ; +run unordered/compile_map.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_compile_map ; +run unordered/copy_tests.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_copy ; +run unordered/move_tests.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_move ; +run unordered/assign_tests.cpp : : : BOOST_UNORDERED_USE_MOVE : bmove_assign ; + +run unordered/serialization_tests.cpp : $(BOOST_UNORDERED_TEST_DIR) - : + : : BOOST_UNORDERED_ENABLE_SERIALIZATION_COMPATIBILITY_V0 off # Boost.Serialization headers are not warning-free norecover:no # boost::archive::xml_oarchive does not pass UBSAN @@ -101,23 +129,49 @@ run unordered/serialization_tests.cpp clang:space /boost//serialization/off ; -run exception/constructor_exception_tests.cpp ; -run exception/copy_exception_tests.cpp ; -run exception/assign_exception_tests.cpp ; -run exception/move_assign_exception_tests.cpp ; -run exception/insert_exception_tests.cpp ; -run exception/erase_exception_tests.cpp ; -run exception/rehash_exception_tests.cpp ; -run exception/swap_exception_tests.cpp : : : BOOST_UNORDERED_SWAP_METHOD=2 ; -run exception/merge_exception_tests.cpp ; -run exception/less_tests.cpp ; +compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_MAP : insert_node_type_fail_map ; +compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_MULTIMAP : insert_node_type_fail_multimap ; +compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_SET : insert_node_type_fail_set ; +compile-fail unordered/insert_node_type_fail.cpp : UNORDERED_TEST_MULTISET : insert_node_type_fail_multiset ; -run unordered/narrow_cast_tests.cpp ; -run quick.cpp ; +local FCA_EXCEPTION_TESTS = + constructor_exception_tests + copy_exception_tests + assign_exception_tests + move_assign_exception_tests + insert_exception_tests + erase_exception_tests + rehash_exception_tests + merge_exception_tests + less_tests + swap_exception_tests +; -import ../../config/checks/config : requires ; +for local test in $(FCA_EXCEPTION_TESTS) +{ + if $(test) = "swap_exception_tests" { + run exception/$(test).cpp : : : BOOST_UNORDERED_SWAP_METHOD=2 ; + } else { + run exception/$(test).cpp ; + } +} -CPP11 = [ requires cxx11_constexpr cxx11_noexcept cxx11_decltype cxx11_alignas ] ; +alias fca_exception_tests : $(FCA_EXCEPTION_TESTS) ; +alias fca_tests : + $(FCA_TESTS) + $(FCA_EXCEPTION_TESTS) + link_test + bmove_compile_set + bmove_compile_map + bmove_copy + bmove_move + bmove_assign + insert_node_type_fail_map + insert_node_type_fail_multimap + insert_node_type_fail_set + insert_node_type_fail_multiset + serialization_tests +; local FOA_TESTS = fwd_set_test @@ -154,21 +208,21 @@ local FOA_TESTS = extract_tests node_handle_tests uses_allocator + hash_is_avalanching_test ; for local test in $(FOA_TESTS) { - run unordered/$(test).cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_$(test) ; + run unordered/$(test).cpp : : : BOOST_UNORDERED_FOA_TESTS : foa_$(test) ; } -run unordered/link_test_1.cpp unordered/link_test_2.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_link_test ; -run unordered/scoped_allocator.cpp : : : $(CPP11) msvc-14.0:no BOOST_UNORDERED_FOA_TESTS : foa_scoped_allocator ; -run unordered/hash_is_avalanching_test.cpp ; +run unordered/link_test_1.cpp unordered/link_test_2.cpp : : : BOOST_UNORDERED_FOA_TESTS : foa_link_test ; +run unordered/scoped_allocator.cpp : : : msvc-14.0:no BOOST_UNORDERED_FOA_TESTS : foa_scoped_allocator ; run unordered/serialization_tests.cpp : - : - : $(CPP11) BOOST_UNORDERED_FOA_TESTS + : + : BOOST_UNORDERED_FOA_TESTS off # Boost.Serialization headers are not warning-free norecover:no # boost::archive::xml_oarchive does not pass UBSAN msvc:/bigobj @@ -179,31 +233,53 @@ run unordered/serialization_tests.cpp /boost//serialization/off : foa_serialization_tests ; -run exception/constructor_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_constructor_exception_tests ; -run exception/copy_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_copy_exception_tests ; -run exception/assign_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_assign_exception_tests ; -run exception/move_assign_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_move_assign_exception_tests ; -run exception/insert_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_insert_exception_tests ; -run exception/erase_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_erase_exception_tests ; -run exception/rehash_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_rehash_exception_tests ; -run exception/swap_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_swap_exception_tests ; -run exception/merge_exception_tests.cpp : : : $(CPP11) BOOST_UNORDERED_FOA_TESTS : foa_merge_exception_tests ; +local FOA_EXCEPTION_TESTS = + constructor_exception_tests + copy_exception_tests + assign_exception_tests + move_assign_exception_tests + insert_exception_tests + erase_exception_tests + rehash_exception_tests + swap_exception_tests + merge_exception_tests +; +for local test in $(FOA_EXCEPTION_TESTS) +{ + run exception/$(test).cpp : : : BOOST_UNORDERED_FOA_TESTS : foa_$(test) ; +} + +local MMAP_CONTAINERS = + unordered_flat_map + unordered_flat_set + unordered_node_map + unordered_node_set + unordered_map + unordered_set + unordered_multimap + unordered_multiset + concurrent_flat_map +; + +for local container in $(MMAP_CONTAINERS) +{ + run unordered/mmap_tests.cpp /boost/filesystem//boost_filesystem : : + : BOOST_UNORDERED_FOA_MMAP_MAP_TYPE="boost::$(container)" + off + static + cygwin:no + : foa_mmap_$(container)_tests ; +} + +alias foa_mmap_tests : foa_mmap_$(MMAP_CONTAINERS)_tests ; alias foa_tests : foa_$(FOA_TESTS) + foa_$(FOA_EXCEPTION_TESTS) foa_link_test foa_scoped_allocator - hash_is_avalanching_test foa_serialization_tests - foa_constructor_exception_tests - foa_copy_exception_tests - foa_assign_exception_tests - foa_move_assign_exception_tests - foa_insert_exception_tests - foa_erase_exception_tests - foa_rehash_exception_tests - foa_swap_exception_tests - foa_merge_exception_tests + foa_mmap_tests ; local CFOA_TESTS = @@ -239,14 +315,14 @@ local CFOA_TESTS = for local test in $(CFOA_TESTS) { run cfoa/$(test).cpp - : requirements $(CPP11) multi + : requirements multi : target-name cfoa_$(test) ; } run cfoa/serialization_tests.cpp : - : + : : $(CPP11) multi off # Boost.Serialization headers are not warning-free norecover:no # boost::archive::xml_oarchive does not pass UBSAN @@ -258,6 +334,6 @@ run cfoa/serialization_tests.cpp /boost//serialization/off : cfoa_serialization_tests ; -alias cfoa_tests : +alias cfoa_tests : cfoa_$(CFOA_TESTS) cfoa_serialization_tests ; diff --git a/test/cfoa/assign_tests.cpp b/test/cfoa/assign_tests.cpp index f2dd2364..409e9d8d 100644 --- a/test/cfoa/assign_tests.cpp +++ b/test/cfoa/assign_tests.cpp @@ -31,7 +31,7 @@ using test::sequential; using hasher = stateful_hash; using key_equal = stateful_key_equal; -using allocator_type = stateful_allocator >; +using allocator_type = stateful_allocator2 >; using flat_map_type = boost::unordered::unordered_flat_map; @@ -847,8 +847,12 @@ namespace { check_raii_counts(); } - template void flat_map_move_assign(G gen, test::random_generator rg) + template + void flat_map_move_assign( + FlatMapType*, MapType*, G gen, test::random_generator rg) { + using alloc_type = typename MapType::allocator_type; + auto values = make_random_values(1024 * 16, [&] { return gen(rg); }); auto reference_map = boost::unordered_flat_map(values.begin(), values.end()); @@ -864,10 +868,10 @@ namespace { { raii::reset_counts(); - flat_map_type flat_map(values.begin(), values.end(), values.size(), - hasher(1), key_equal(2), allocator_type(3)); + FlatMapType flat_map(values.begin(), values.end(), values.size(), + hasher(1), key_equal(2), alloc_type(3)); - map_type map(0, hasher(2), key_equal(1), allocator_type(3)); + MapType map(0, hasher(2), key_equal(1), alloc_type(3)); BOOST_TEST(flat_map.get_allocator() == map.get_allocator()); @@ -893,10 +897,10 @@ namespace { { raii::reset_counts(); - map_type map(values.begin(), values.end(), values.size(), hasher(1), - key_equal(2), allocator_type(3)); + MapType map(values.begin(), values.end(), values.size(), hasher(1), + key_equal(2), alloc_type(3)); - flat_map_type flat_map(0, hasher(2), key_equal(1), allocator_type(3)); + FlatMapType flat_map(0, hasher(2), key_equal(1), alloc_type(3)); BOOST_TEST(flat_map.get_allocator() == map.get_allocator()); @@ -920,10 +924,10 @@ namespace { { raii::reset_counts(); - flat_map_type flat_map(values.begin(), values.end(), values.size(), - hasher(1), key_equal(2), allocator_type(3)); + FlatMapType flat_map(values.begin(), values.end(), values.size(), + hasher(1), key_equal(2), alloc_type(3)); - map_type map(0, hasher(2), key_equal(1), allocator_type(4)); + MapType map(0, hasher(2), key_equal(1), alloc_type(4)); BOOST_TEST(flat_map.get_allocator() != map.get_allocator()); @@ -950,10 +954,10 @@ namespace { { raii::reset_counts(); - map_type map(values.begin(), values.end(), values.size(), hasher(1), - key_equal(2), allocator_type(3)); + MapType map(values.begin(), values.end(), values.size(), hasher(1), + key_equal(2), alloc_type(3)); - flat_map_type flat_map(0, hasher(2), key_equal(1), allocator_type(4)); + FlatMapType flat_map(0, hasher(2), key_equal(1), alloc_type(4)); BOOST_TEST(flat_map.get_allocator() != map.get_allocator()); @@ -994,8 +998,27 @@ UNORDERED_TEST( ((init_type_generator)) ((default_generator)(sequential)(limited_range))) +boost::unordered::unordered_flat_map > >* flat_map_plain; +boost::unordered::unordered_flat_map > >* flat_map_fancy; + +boost::unordered::concurrent_flat_map > >* map_plain; +boost::unordered::concurrent_flat_map > >* map_fancy; + UNORDERED_TEST( flat_map_move_assign, + ((flat_map_plain)) + ((map_plain)) + ((init_type_generator)) + ((default_generator)(sequential)(limited_range))) + +UNORDERED_TEST( + flat_map_move_assign, + ((flat_map_fancy)) + ((map_fancy)) ((init_type_generator)) ((default_generator)(sequential)(limited_range))) // clang-format on diff --git a/test/cfoa/helpers.hpp b/test/cfoa/helpers.hpp index 02ecfa05..79b91bb0 100644 --- a/test/cfoa/helpers.hpp +++ b/test/cfoa/helpers.hpp @@ -123,6 +123,27 @@ struct stateful_key_equal } }; +template struct cfoa_ptr +{ +private: + template friend struct stateful_allocator2; + + T* p_ = nullptr; + + cfoa_ptr(T* p) : p_(p) {} + +public: + using element_type = T; + + cfoa_ptr() = default; + cfoa_ptr(std::nullptr_t) : p_(nullptr){}; + template using rebind = cfoa_ptr; + + T* operator->() const noexcept { return p_; } + + static cfoa_ptr pointer_to(element_type& r) { return {std::addressof(r)}; } +}; + template struct stateful_allocator { int x_ = -1; @@ -151,6 +172,36 @@ template struct stateful_allocator bool operator!=(stateful_allocator const& rhs) const { return x_ != rhs.x_; } }; +template struct stateful_allocator2 +{ + + int x_ = -1; + + using value_type = T; + using pointer = cfoa_ptr; + + stateful_allocator2() = default; + stateful_allocator2(stateful_allocator2 const&) = default; + stateful_allocator2(stateful_allocator2&&) = default; + + stateful_allocator2(int const x) : x_{x} {} + + template + stateful_allocator2(stateful_allocator2 const& rhs) : x_{rhs.x_} + { + } + + pointer allocate(std::size_t n) + { + return {static_cast(::operator new(n * sizeof(T)))}; + } + + void deallocate(pointer p, std::size_t) { ::operator delete(p.p_); } + + bool operator==(stateful_allocator2 const& rhs) const { return x_ == rhs.x_; } + bool operator!=(stateful_allocator2 const& rhs) const { return x_ != rhs.x_; } +}; + struct raii { static std::atomic default_constructor; @@ -458,6 +509,7 @@ template class ptr public: ptr() : ptr_(0) {} + ptr(std::nullptr_t) : ptr_(nullptr) {} explicit ptr(void_ptr const& x) : ptr_((T*)x.ptr_) {} T& operator*() const { return *ptr_; } diff --git a/test/objects/minimal.hpp b/test/objects/minimal.hpp index c2690db3..bf94d924 100644 --- a/test/objects/minimal.hpp +++ b/test/objects/minimal.hpp @@ -307,6 +307,7 @@ namespace test { public: ptr() : ptr_(0) {} + ptr(std::nullptr_t) : ptr_(0) {} explicit ptr(void_ptr const& x) : ptr_((T*)x.ptr_) {} T& operator*() const { return *ptr_; } @@ -325,6 +326,18 @@ namespace test { ptr operator+(std::ptrdiff_t s) const { return ptr(ptr_ + s); } friend ptr operator+(std::ptrdiff_t s, ptr p) { return ptr(s + p.ptr_); } + ptr& operator+=(std::ptrdiff_t s) + { + ptr_ += s; + return *this; + } + + ptr& operator-=(std::ptrdiff_t s) + { + ptr_ -= s; + return *this; + } + std::ptrdiff_t operator-(ptr p) const { return ptr_ - p.ptr_; } ptr operator-(std::ptrdiff_t s) const { return ptr(ptr_ - s); } T& operator[](std::ptrdiff_t s) const { return ptr_[s]; } @@ -340,6 +353,8 @@ namespace test { bool operator==(ptr const& x) const { return ptr_ == x.ptr_; } bool operator!=(ptr const& x) const { return ptr_ != x.ptr_; } + bool operator==(std::nullptr_t) const { return ptr_ == nullptr; } + bool operator!=(std::nullptr_t) const { return ptr_ != nullptr; } bool operator<(ptr const& x) const { return ptr_ < x.ptr_; } bool operator>(ptr const& x) const { return ptr_ > x.ptr_; } bool operator<=(ptr const& x) const { return ptr_ <= x.ptr_; } @@ -660,6 +675,9 @@ namespace boost { { typedef ::test::minimal::ptr type; }; + + template + using rebind=typename rebind_to::type; }; } diff --git a/test/objects/test.hpp b/test/objects/test.hpp index 853e2205..bb2b2896 100644 --- a/test/objects/test.hpp +++ b/test/objects/test.hpp @@ -16,6 +16,9 @@ #include #include +template struct allocator1; +template struct allocator2; + namespace test { // Note that the default hash function will work for any equal_to (but not // very well). @@ -506,14 +509,18 @@ namespace test { template class ptr { + friend struct ::allocator1; + friend struct ::allocator2; friend class allocator2; friend class const_ptr; friend struct void_ptr; - public: T* ptr_; ptr(T* x) : ptr_(x) {} + + public: ptr() : ptr_(0) {} + ptr(std::nullptr_t) : ptr_(nullptr) {} explicit ptr(void_ptr const& x) : ptr_((T*)x.ptr_) {} T& operator*() const { return *ptr_; } @@ -537,6 +544,7 @@ namespace test { ptr operator-(std::ptrdiff_t s) const { return ptr(ptr_ - s); } ptr& operator+=(std::ptrdiff_t s) { ptr_ += s; return *this; } + ptr& operator-=(std::ptrdiff_t s) { ptr_ -= s; return *this; } T& operator[](std::ptrdiff_t s) const { return ptr_[s]; } bool operator!() const { return !ptr_; } @@ -727,6 +735,9 @@ namespace boost { { typedef ::test::ptr type; }; + + template + using rebind=typename rebind_to::type; }; } // namespace boost diff --git a/test/unordered/assign_tests.cpp b/test/unordered/assign_tests.cpp index 8340ac20..e75f734d 100644 --- a/test/unordered/assign_tests.cpp +++ b/test/unordered/assign_tests.cpp @@ -41,7 +41,6 @@ namespace assign_tests { BOOST_TEST(x.empty()); BOOST_TEST(test::equivalent(x.hash_function(), hf)); BOOST_TEST(test::equivalent(x.key_eq(), eq)); - BOOST_TEST(test::detail::tracker.count_allocations == 0); } BOOST_LIGHTWEIGHT_TEST_OSTREAM << "assign_tests1.2\n"; @@ -91,7 +90,6 @@ namespace assign_tests { BOOST_TEST(test::equivalent(x1.key_eq(), eq1)); BOOST_TEST(test::equivalent(x2.hash_function(), hf1)); BOOST_TEST(test::equivalent(x2.key_eq(), eq1)); - BOOST_TEST(test::detail::tracker.count_allocations == 0); test::check_container(x1, x2); } diff --git a/test/unordered/constructor_tests.cpp b/test/unordered/constructor_tests.cpp index e65ebfb3..54268a42 100644 --- a/test/unordered/constructor_tests.cpp +++ b/test/unordered/constructor_tests.cpp @@ -6,13 +6,13 @@ #include "../helpers/unordered.hpp" -#include "../helpers/test.hpp" -#include "../objects/test.hpp" -#include "../helpers/random_values.hpp" -#include "../helpers/tracker.hpp" #include "../helpers/equivalent.hpp" #include "../helpers/input_iterator.hpp" #include "../helpers/invariants.hpp" +#include "../helpers/random_values.hpp" +#include "../helpers/test.hpp" +#include "../helpers/tracker.hpp" +#include "../objects/test.hpp" #include @@ -33,7 +33,6 @@ namespace constructor_tests { T x(0, hf, eq); BOOST_TEST(x.empty()); - BOOST_TEST_EQ(x.bucket_count(), 0u); BOOST_TEST(test::equivalent(x.hash_function(), hf)); BOOST_TEST(test::equivalent(x.key_eq(), eq)); BOOST_TEST(test::equivalent(x.get_allocator(), al)); @@ -72,7 +71,6 @@ namespace constructor_tests { T x; BOOST_TEST(x.empty()); - BOOST_TEST_EQ(x.bucket_count(), 0u); BOOST_TEST(test::equivalent(x.hash_function(), hf)); BOOST_TEST(test::equivalent(x.key_eq(), eq)); BOOST_TEST(test::equivalent(x.get_allocator(), al)); @@ -140,7 +138,6 @@ namespace constructor_tests { T x(0, hf, eq, al); BOOST_TEST(x.empty()); - BOOST_TEST_EQ(x.bucket_count(), 0u); BOOST_TEST(test::equivalent(x.hash_function(), hf)); BOOST_TEST(test::equivalent(x.key_eq(), eq)); BOOST_TEST(test::equivalent(x.get_allocator(), al)); @@ -167,7 +164,6 @@ namespace constructor_tests { T x(al); BOOST_TEST(x.empty()); - BOOST_TEST_EQ(x.bucket_count(), 0u); BOOST_TEST(test::equivalent(x.hash_function(), hf)); BOOST_TEST(test::equivalent(x.key_eq(), eq)); BOOST_TEST(test::equivalent(x.get_allocator(), al)); @@ -349,7 +345,6 @@ namespace constructor_tests { { T x(list); BOOST_TEST(x.empty()); - BOOST_TEST_EQ(x.bucket_count(), 0u); BOOST_TEST(test::equivalent(x.hash_function(), hf)); BOOST_TEST(test::equivalent(x.key_eq(), eq)); BOOST_TEST(test::equivalent(x.get_allocator(), al)); @@ -536,6 +531,16 @@ namespace constructor_tests { template void no_alloc_default_construct_test(T*, test::random_generator) { + +#ifdef BOOST_UNORDERED_FOA_TESTS + using allocator_type = typename T::allocator_type; + using value_type = + typename boost::allocator_value_type::type; + using pointer = typename boost::allocator_pointer::type; + static_assert(std::is_same::value, + "only raw pointers for this test"); +#endif + UNORDERED_SUB_TEST("Construct 1") { T x; @@ -653,6 +658,15 @@ namespace constructor_tests { boost::unordered_flat_map >* test_map_std_alloc; + boost::unordered_flat_set >* test_set_raw_ptr; + boost::unordered_node_set >* test_node_set_raw_ptr; + boost::unordered_flat_map >* test_map_raw_ptr; + boost::unordered_node_map >* test_node_map_raw_ptr; + boost::unordered_flat_set >* test_set; boost::unordered_node_set void max_load_tests(X*, test::random_generator generator) test::reset_sequence(); X x; - size_type max_load = x.max_load(); - - BOOST_TEST_EQ(max_load, 0u); x.reserve(1000); - max_load = x.max_load(); + size_type max_load = x.max_load(); + + BOOST_TEST_GT(max_load, 0u); size_type bucket_count = x.bucket_count(); BOOST_TEST_GE(bucket_count, 1000u); diff --git a/test/unordered/mmap_tests.cpp b/test/unordered/mmap_tests.cpp new file mode 100644 index 00000000..1e900bc6 --- /dev/null +++ b/test/unordered/mmap_tests.cpp @@ -0,0 +1,312 @@ +// Copyright 2023 Christian Mazakas. +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(BOOST_CLANG_VERSION) && BOOST_CLANG_VERSION < 30900 +#include +BOOST_PRAGMA_MESSAGE( + "This version of clang is incompatible with Boost.Process"); +int main() {} +#else +#include "../helpers/test.hpp" + +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include + +#ifndef BOOST_UNORDERED_FOA_MMAP_MAP_TYPE +#error "this requires a class template be passed as a macro" +#endif + +using char_allocator = boost::interprocess::allocator; + +using string_type = boost::interprocess::basic_string, char_allocator>; + +using pair_type = std::pair; + +using string_pair_type = std::pair; + +using string_pair_allocator = boost::interprocess::allocator; + +using pair_allocator = boost::interprocess::allocator; + +template