From 886b1b4deda5c59d118d70e628e9eed6fe6e9c94 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Wed, 5 Oct 2022 13:54:28 +0200 Subject: [PATCH] refactored table_arrays to solve alignment issues without an extra data member --- include/boost/unordered/detail/foa.hpp | 175 +++++++------------------ 1 file changed, 49 insertions(+), 126 deletions(-) diff --git a/include/boost/unordered/detail/foa.hpp b/include/boost/unordered/detail/foa.hpp index 08a60d76..cf5e82c7 100644 --- a/include/boost/unordered/detail/foa.hpp +++ b/include/boost/unordered/detail/foa.hpp @@ -668,22 +668,31 @@ private: Value *p=nullptr; }; -template -struct table_arrays_base +template +struct table_arrays { + using value_type=Value; + using group_type=Group; + static constexpr auto N=group_type::N; + using size_policy=SizePolicy; + template - static Arrays new_(Allocator& al,std::size_t n) + static table_arrays new_(Allocator& al,std::size_t n) { - using group_type=typename Arrays::group_type; - static constexpr auto N=group_type::N; - using size_policy=typename Arrays::size_policy; using alloc_traits=std::allocator_traits; - auto groups_size_index=size_policy::size_index(n/N+1); - auto groups_size=size_policy::size(groups_size_index); - Arrays arrays{groups_size_index,groups_size-1}; + /* n/N+1 == ceil(n+1/N) (extra +1 for the sentinel) */ + auto groups_size_index=size_policy::size_index(n/N+1); + auto groups_size=size_policy::size(groups_size_index); + table_arrays arrays{groups_size_index,groups_size-1,nullptr,nullptr}; if(!n){ + /* We make groups point to dummy storage initialized as if in an empty + * container. This allows us to implement find() etc. without checking + * groups==nullptr. This space won't ever be used for insertion as + * container capacity is properly tuned to avoid that. + */ + static constexpr typename group_type::dummy_group_type storage[size_policy::min_size()]= {typename group_type::dummy_group_type(),}; @@ -692,145 +701,59 @@ struct table_arrays_base const_cast(storage)); } else{ - arrays.allocate_groups(al,groups_size); - // TODO: explain why memset - std::memset( - arrays.groups,0,sizeof(group_type)*groups_size); + arrays.elements= + boost::to_address(alloc_traits::allocate(al,buffer_size(groups_size))); + + /* Align arrays.groups to sizeof(group_type). table_iterator critically + * depends on such alignment for its increment operation. + */ + + auto p=reinterpret_cast(arrays.elements+groups_size*N-1); + p+=(uintptr_t(sizeof(group_type))- + reinterpret_cast(p))%sizeof(group_type); + arrays.groups=reinterpret_cast(p); + + /* memset is faster/not slower than using group_type default ctor. + * This assumes all zeros is group_type's default layout. + */ + + std::memset(arrays.groups,0,sizeof(group_type)*groups_size); arrays.groups[groups_size-1].set_sentinel(); - BOOST_TRY{ - arrays.elements= - boost::to_address(alloc_traits::allocate(al,groups_size*N-1)); - } - BOOST_CATCH(...){ - arrays.deallocate_groups(al,groups_size); - BOOST_RETHROW - } - BOOST_CATCH_END } return arrays; } template - static void delete_(Allocator& al,Arrays& arrays)noexcept + static void delete_(Allocator& al,table_arrays& arrays)noexcept { - using group_type=typename Arrays::group_type; - static constexpr auto N=group_type::N; using alloc_traits=std::allocator_traits; if(arrays.elements){ - auto groups_size=arrays.groups_size_mask+1; - alloc_traits::deallocate(al,arrays.elements,groups_size*N-1); - arrays.deallocate_groups(al,groups_size); + alloc_traits::deallocate( + al,arrays.elements,buffer_size(arrays.groups_size_mask+1)); } } -}; -template -struct aligned_table_arrays: - table_arrays_base> -{ - using group_type=Group; - using value_type=Value; - using size_policy=SizePolicy; + /* combined space for elements and groups measured in sizeof(value_type)s */ - aligned_table_arrays( - std::size_t groups_size_index_,std::size_t groups_size_mask_): - groups_size_index{groups_size_index_},groups_size_mask{groups_size_mask_} - {} - - template - void allocate_groups(Allocator& al,std::size_t groups_size) + static std::size_t buffer_size(std::size_t groups_size) { - using alloc_traits=std::allocator_traits; - using group_allocator= - typename alloc_traits::template rebind_alloc; - using group_alloc_traits=std::allocator_traits; + auto buffer_bytes= + /* space for elements (we subtract 1 because of the sentinel) */ + sizeof(value_type)*(groups_size*N-1)+ + /* space for groups + padding for group alignment */ + sizeof(group_type)*(groups_size+1)-1; - group_allocator gal=al; - groups=boost::to_address(group_alloc_traits::allocate(gal,groups_size)); - } - - template - void deallocate_groups(Allocator& al,std::size_t groups_size) - { - using alloc_traits=std::allocator_traits; - using group_allocator= - typename alloc_traits::template rebind_alloc; - using group_alloc_traits=std::allocator_traits; - - group_allocator gal=al; - group_alloc_traits::deallocate(gal,groups,groups_size); + /* ceil(buffer_bytes/sizeof(value_type)) */ + return (buffer_bytes+sizeof(value_type)-1)/sizeof(value_type); } std::size_t groups_size_index; std::size_t groups_size_mask; - group_type *groups=nullptr; - value_type *elements=nullptr; + group_type *groups; + value_type *elements; }; -template -struct subaligned_table_arrays: - table_arrays_base> -{ - using group_type=Group; - using value_type=Value; - using size_policy=SizePolicy; - - subaligned_table_arrays( - std::size_t groups_size_index_,std::size_t groups_size_mask_): - groups_size_index{groups_size_index_},groups_size_mask{groups_size_mask_} - {} - - template - void allocate_groups(Allocator& al,std::size_t groups_size) - { - using alloc_traits=std::allocator_traits; - using byte_allocator= - typename alloc_traits::template rebind_alloc; - using byte_alloc_traits=std::allocator_traits; - - byte_allocator bal=al; - auto p=boost::to_address( - byte_alloc_traits::allocate(bal,sizeof(group_type)*(groups_size+1)-1)); - groups_offset=static_cast( - (uintptr_t(sizeof(group_type))-reinterpret_cast(p))% - sizeof(group_type)); - groups=reinterpret_cast(p+groups_offset); - } - - template - void deallocate_groups(Allocator& al,std::size_t groups_size) - { - using alloc_traits=std::allocator_traits; - using byte_allocator= - typename alloc_traits::template rebind_alloc; - using byte_alloc_traits=std::allocator_traits; - - byte_allocator bal=al; - byte_alloc_traits::deallocate( - bal,reinterpret_cast(groups)-groups_offset, - sizeof(group_type)*(groups_size+1)-1); - } - - std::size_t groups_size_index; - std::size_t groups_size_mask; - group_type *groups=nullptr; - value_type *elements=nullptr; - unsigned char groups_offset=0; -}; - -template -using table_arrays=typename std::conditional< - -#if 0&&defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__) - sizeof(Group)<=__STDCPP_DEFAULT_NEW_ALIGNMENT__, -#else - sizeof(Group)<=alignof(std::max_align_t), -#endif - - aligned_table_arrays, - subaligned_table_arrays>::type; - struct if_constexpr_void_else{void operator()()const{}}; template