boostorg/unordered
1
0
Fork 1
mirror of https://github.com/boostorg/unordered.git synced 2025-10-28 06:11:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

Feature/bulk visit (#217)

Browse Source
This commit is contained in:
joaquintides
2023-10-11 12:50:28 +02:00
committed by GitHub
parent ef0b3a0cd8
commit 8ee48fe909
54 changed files with 507 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
include/boost/unordered/concurrent_flat_map.hpp
View File

@@ -1,6 +1,7 @@
/* Fast open-addressing concurrent hashmap.
*
* Copyright 2023 Christian Mazakas.
* Copyright 2023 Joaquin M Lopez Munoz.
* Distributed under the Boost Software License, Version 1.0.
* (See accompanying file LICENSE_1_0.txt or copy at
* http://www.boost.org/LICENSE_1_0.txt)
@@ -72,6 +73,7 @@ namespace boost {
using pointer = typename boost::allocator_pointer<allocator_type>::type;
using const_pointer =
typename boost::allocator_const_pointer<allocator_type>::type;
static constexpr size_type bulk_visit_size = table_type::bulk_visit_size;
concurrent_flat_map()
: concurrent_flat_map(detail::foa::default_bucket_count)
@@ -270,6 +272,33 @@ namespace boost {
return table_.visit(std::forward<K>(k), f);
}
template<class FwdIterator, class F>
BOOST_FORCEINLINE
size_t visit(FwdIterator first, FwdIterator last, F f)
{
BOOST_UNORDERED_STATIC_ASSERT_BULK_VISIT_ITERATOR(FwdIterator)
BOOST_UNORDERED_STATIC_ASSERT_INVOCABLE(F)
return table_.visit(first, last, f);
}
template<class FwdIterator, class F>
BOOST_FORCEINLINE
size_t visit(FwdIterator first, FwdIterator last, F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_BULK_VISIT_ITERATOR(FwdIterator)
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
return table_.visit(first, last, f);
}
template<class FwdIterator, class F>
BOOST_FORCEINLINE
size_t cvisit(FwdIterator first, FwdIterator last, F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_BULK_VISIT_ITERATOR(FwdIterator)
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
return table_.visit(first, last, f);
}
template <class F> size_type visit_all(F f)
{
BOOST_UNORDERED_STATIC_ASSERT_INVOCABLE(F)

24
include/boost/unordered/concurrent_flat_set.hpp
View File

@@ -38,7 +38,10 @@ namespace boost {
using type_policy = detail::foa::flat_set_types<Key>;
detail::foa::concurrent_table<type_policy, Hash, Pred, Allocator> table_;
using table_type =
detail::foa::concurrent_table<type_policy, Hash, Pred, Allocator>;
table_type table_;
template <class K, class H, class KE, class A>
bool friend operator==(concurrent_flat_set<K, H, KE, A> const& lhs,
@@ -67,6 +70,7 @@ namespace boost {
using pointer = typename boost::allocator_pointer<allocator_type>::type;
using const_pointer =
typename boost::allocator_const_pointer<allocator_type>::type;
static constexpr size_type bulk_visit_size = table_type::bulk_visit_size;
concurrent_flat_set()
: concurrent_flat_set(detail::foa::default_bucket_count)
@@ -251,6 +255,24 @@ namespace boost {
return table_.visit(std::forward<K>(k), f);
}
template<class FwdIterator, class F>
BOOST_FORCEINLINE
size_t visit(FwdIterator first, FwdIterator last, F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_BULK_VISIT_ITERATOR(FwdIterator)
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
return table_.visit(first, last, f);
}
template<class FwdIterator, class F>
BOOST_FORCEINLINE
size_t cvisit(FwdIterator first, FwdIterator last, F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_BULK_VISIT_ITERATOR(FwdIterator)
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
return table_.visit(first, last, f);
}
template <class F> size_type visit_all(F f) const
{
BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)

30
include/boost/unordered/detail/concurrent_static_asserts.hpp
View File

@@ -10,10 +10,12 @@
#ifndef BOOST_UNORDERED_DETAIL_CONCURRENT_STATIC_ASSERTS_HPP
#define BOOST_UNORDERED_DETAIL_CONCURRENT_STATIC_ASSERTS_HPP
#include <boost/config.hpp>
#include <boost/mp11/algorithm.hpp>
#include <boost/mp11/list.hpp>
#include <functional>
#include <iterator>
#include <type_traits>
#define BOOST_UNORDERED_STATIC_ASSERT_INVOCABLE(F) \
@@ -72,4 +74,32 @@ namespace boost {
} // namespace boost
#if defined(BOOST_NO_CXX20_HDR_CONCEPTS)
#define BOOST_UNORDERED_STATIC_ASSERT_FWD_ITERATOR(Iterator) \
static_assert( \
std::is_base_of< \
std::forward_iterator_tag, \
typename std::iterator_traits<Iterator>::iterator_category>::value, \
"The provided iterator must be at least forward");
#else
#define BOOST_UNORDERED_STATIC_ASSERT_FWD_ITERATOR(Iterator) \
static_assert(std::forward_iterator<Iterator>, \
"The provided iterator must be at least forward");
#endif
#define BOOST_UNORDERED_STATIC_ASSERT_KEY_COMPATIBLE_ITERATOR(Iterator) \
static_assert( \
std::is_same< \
typename std::iterator_traits<Iterator>::value_type, \
key_type>::value || \
detail::are_transparent< \
typename std::iterator_traits<Iterator>::value_type, \
hasher, key_equal>::value, \
"The provided iterator must dereference to a compatible key value");
#define BOOST_UNORDERED_STATIC_ASSERT_BULK_VISIT_ITERATOR(Iterator) \
BOOST_UNORDERED_STATIC_ASSERT_FWD_ITERATOR(Iterator) \
BOOST_UNORDERED_STATIC_ASSERT_KEY_COMPATIBLE_ITERATOR(Iterator)
#endif // BOOST_UNORDERED_DETAIL_CONCURRENT_STATIC_ASSERTS_HPP

113
include/boost/unordered/detail/foa/concurrent_table.hpp
View File

@@ -31,6 +31,7 @@
#include <cstddef>
#include <functional>
#include <initializer_list>
#include <iterator>
#include <memory>
#include <new>
#include <type_traits>
@@ -465,6 +466,7 @@ public:
using key_equal=typename super::key_equal;
using allocator_type=typename super::allocator_type;
using size_type=typename super::size_type;
static constexpr std::size_t bulk_visit_size=16;
private:
template<typename Value,typename T>
@@ -564,6 +566,27 @@ public:
return visit(x,std::forward<F>(f));
}
template<typename FwdIterator,typename F>
BOOST_FORCEINLINE
std::size_t visit(FwdIterator first,FwdIterator last,F&& f)
{
return bulk_visit_impl(group_exclusive{},first,last,std::forward<F>(f));
}
template<typename FwdIterator,typename F>
BOOST_FORCEINLINE
std::size_t visit(FwdIterator first,FwdIterator last,F&& f)const
{
return bulk_visit_impl(group_shared{},first,last,std::forward<F>(f));
}
template<typename FwdIterator,typename F>
BOOST_FORCEINLINE
std::size_t cvisit(FwdIterator first,FwdIterator last,F&& f)const
{
return visit(first,last,std::forward<F>(f));
}
template<typename F> std::size_t visit_all(F&& f)
{
return visit_all_impl(group_exclusive{},std::forward<F>(f));
@@ -1051,6 +1074,26 @@ private:
access_mode,x,this->position_for(hash),hash,std::forward<F>(f));
}
template<typename GroupAccessMode,typename FwdIterator,typename F>
BOOST_FORCEINLINE
std::size_t bulk_visit_impl(
GroupAccessMode access_mode,FwdIterator first,FwdIterator last,F&& f)const
{
auto lck=shared_access();
std::size_t res=0;
auto n=static_cast<std::size_t>(std::distance(first,last));
while(n){
auto m=n<2*bulk_visit_size?n:bulk_visit_size;
res+=unprotected_bulk_visit(access_mode,first,m,std::forward<F>(f));
n-=m;
std::advance(
first,
static_cast<
typename std::iterator_traits<FwdIterator>::difference_type>(m));
}
return res;
}
template<typename GroupAccessMode,typename F>
std::size_t visit_all_impl(GroupAccessMode access_mode,F&& f)const
{
@@ -1149,6 +1192,76 @@ private:
return 0;
}
template<typename GroupAccessMode,typename FwdIterator,typename F>
BOOST_FORCEINLINE std::size_t unprotected_bulk_visit(
GroupAccessMode access_mode,FwdIterator first,std::size_t m,F&& f)const
{
BOOST_ASSERT(m<2*bulk_visit_size);
std::size_t res=0,
hashes[2*bulk_visit_size-1],
positions[2*bulk_visit_size-1];
int masks[2*bulk_visit_size-1];
auto it=first;
for(auto i=m;i--;++it){
auto hash=hashes[i]=this->hash_for(*it);
auto pos=positions[i]=this->position_for(hash);
BOOST_UNORDERED_PREFETCH(this->arrays.groups()+pos);
}
for(auto i=m;i--;){
auto hash=hashes[i];
auto pos=positions[i];
auto mask=masks[i]=(this->arrays.groups()+pos)->match(hash);
if(mask){
BOOST_UNORDERED_PREFETCH(this->arrays.group_accesses()+pos);
BOOST_UNORDERED_PREFETCH_ELEMENTS(this->arrays.elements()+pos*N,N);
}
}
it=first;
for(auto i=m;i--;++it){
auto pos=positions[i];
prober pb(pos);
auto pg=this->arrays.groups()+pos;
auto mask=masks[i];
element_type *p;
if(!mask)goto post_mask;
p=this->arrays.elements()+pos*N;
for(;;){
{
auto lck=access(access_mode,pos);
do{
auto n=unchecked_countr_zero(mask);
if(BOOST_LIKELY(
pg->is_occupied(n)&&
bool(this->pred()(*it,this->key_from(p[n]))))){
f(cast_for(access_mode,type_policy::value_from(p[n])));
++res;
goto next_key;
}
mask&=mask-1;
}while(mask);
}
post_mask:
do{
if(BOOST_LIKELY(pg->is_not_overflowed(hashes[i]))||
BOOST_UNLIKELY(!pb.next(this->arrays.groups_size_mask))){
goto next_key;
}
pos=pb.get();
pg=this->arrays.groups()+pos;
mask=pg->match(hashes[i]);
}while(!mask);
p=this->arrays.elements()+pos*N;
BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
}
next_key:;
}
return res;
}
#if defined(BOOST_MSVC)
#pragma warning(pop) /* C4800 */
#endif

2
include/boost/unordered/detail/foa/core.hpp
View File

@@ -92,7 +92,7 @@
#elif defined(BOOST_UNORDERED_SSE2)
#define BOOST_UNORDERED_PREFETCH(p) _mm_prefetch((const char*)(p),_MM_HINT_T0)
#else
#define BOOST_UNORDERED_PREFETCH(p) ((void)0)
#define BOOST_UNORDERED_PREFETCH(p) ((void)(p))
#endif
/* We have experimentally confirmed that ARM architectures get a higher