streamlined unprotected_bulk_visit

include/boost/unordered/concurrent_flat_map.hpp

@@ -244,6 +244,14 @@ namespace boost {
         return table_.visit(k, f);
       }
 
+      template<typename FwdIterator,typename F>
+      BOOST_FORCEINLINE
+      std::size_t visit(FwdIterator first,FwdIterator last,F f)const
+      {
+        BOOST_UNORDERED_STATIC_ASSERT_CONST_INVOCABLE(F)
+        return table_.visit(first, last, f);
+      }
+
       template <class K, class F>
       BOOST_FORCEINLINE typename std::enable_if<
         detail::are_transparent<K, hasher, key_equal>::value, size_type>::type

include/boost/unordered/detail/foa/concurrent_table.hpp

@@ -31,6 +31,7 @@
 #include <cstddef>
 #include <functional>
 #include <initializer_list>
+#include <iterator>
 #include <memory>
 #include <new>
 #include <type_traits>
@@ -564,6 +565,97 @@ public:
     return visit(x,std::forward<F>(f));
   }
 
+  static constexpr std::size_t bulk_visit_size=16;
+
+#if 0
+#define BOOST_UNORDERED_BULK_VISIT_CASE(r,b)  \
+case r:                                       \
+  res+=unprotected_bulk_visit<b>(             \
+    group_shared{},first,std::forward<F>(f)); \
+  break;
+
+  template<typename FwdIterator,typename F>
+  std::size_t visit(FwdIterator first,FwdIterator last,F&& f)const
+  {
+    BOOST_STATIC_ASSERT(bulk_visit_size==16);
+
+    std::size_t res=0;
+    auto        n=static_cast<std::size_t>(std::distance(first,last));
+    auto        m=n/bulk_visit_size;
+    auto        lck=shared_access();
+
+    if(m){
+      for(std::size_t i=m-1;i--;){
+        res+=unprotected_bulk_visit<bulk_visit_size>(
+          group_shared{},first,std::forward<F>(f));
+        std::advance(first,bulk_visit_size);
+      }
+
+      switch(n-m*bulk_visit_size){
+        BOOST_UNORDERED_BULK_VISIT_CASE(0,bulk_visit_size)
+        BOOST_UNORDERED_BULK_VISIT_CASE(1,bulk_visit_size+1)
+        BOOST_UNORDERED_BULK_VISIT_CASE(2,bulk_visit_size+2)
+        BOOST_UNORDERED_BULK_VISIT_CASE(3,bulk_visit_size+3)
+        BOOST_UNORDERED_BULK_VISIT_CASE(4,bulk_visit_size+4)
+        BOOST_UNORDERED_BULK_VISIT_CASE(5,bulk_visit_size+5)
+        BOOST_UNORDERED_BULK_VISIT_CASE(6,bulk_visit_size+6)
+        BOOST_UNORDERED_BULK_VISIT_CASE(7,bulk_visit_size+7)
+        BOOST_UNORDERED_BULK_VISIT_CASE(8,bulk_visit_size+8)
+        BOOST_UNORDERED_BULK_VISIT_CASE(9,bulk_visit_size+9)
+        BOOST_UNORDERED_BULK_VISIT_CASE(10,bulk_visit_size+10)
+        BOOST_UNORDERED_BULK_VISIT_CASE(11,bulk_visit_size+11)
+        BOOST_UNORDERED_BULK_VISIT_CASE(12,bulk_visit_size+12)
+        BOOST_UNORDERED_BULK_VISIT_CASE(13,bulk_visit_size+13)
+        BOOST_UNORDERED_BULK_VISIT_CASE(14,bulk_visit_size+14)
+        BOOST_UNORDERED_BULK_VISIT_CASE(15,bulk_visit_size+15)
+        default:break;
+      }
+    }
+    else{
+      switch(n){
+        case 0:break;
+        BOOST_UNORDERED_BULK_VISIT_CASE(1,1)
+        BOOST_UNORDERED_BULK_VISIT_CASE(2,2)
+        BOOST_UNORDERED_BULK_VISIT_CASE(3,3)
+        BOOST_UNORDERED_BULK_VISIT_CASE(4,4)
+        BOOST_UNORDERED_BULK_VISIT_CASE(5,5)
+        BOOST_UNORDERED_BULK_VISIT_CASE(6,6)
+        BOOST_UNORDERED_BULK_VISIT_CASE(7,7)
+        BOOST_UNORDERED_BULK_VISIT_CASE(8,8)
+        BOOST_UNORDERED_BULK_VISIT_CASE(9,9)
+        BOOST_UNORDERED_BULK_VISIT_CASE(10,10)
+        BOOST_UNORDERED_BULK_VISIT_CASE(11,11)
+        BOOST_UNORDERED_BULK_VISIT_CASE(12,12)
+        BOOST_UNORDERED_BULK_VISIT_CASE(13,13)
+        BOOST_UNORDERED_BULK_VISIT_CASE(14,14)
+        BOOST_UNORDERED_BULK_VISIT_CASE(15,15)
+        default:break;
+      }
+    }
+    return res;
+  }
+
+#undef BOOST_UNORDERED_BULK_VISIT_CASE
+#else
+  template<typename FwdIterator,typename F>
+  BOOST_FORCEINLINE
+  std::size_t visit(FwdIterator first,FwdIterator last,F&& f)const
+  {
+    std::size_t res=0;
+    auto        n=static_cast<std::size_t>(std::distance(first,last));
+    auto        lck=shared_access();
+
+    while(n){
+      auto m=n<2*bulk_visit_size?n:bulk_visit_size;
+      res+=unprotected_bulk_visit(group_shared{},first,m,std::forward<F>(f));
+      n-=m;
+      std::advance(first,m);
+    }
+
+    return res;
+  }
+#endif
+
   template<typename F> std::size_t visit_all(F&& f)
   {
     return visit_all_impl(group_exclusive{},std::forward<F>(f));
@@ -1149,6 +1241,153 @@ private:
     return 0;
   }
 
+#if 0
+  template<
+    std::size_t M,typename GroupAccessMode,
+    typename FwdIterator,typename F
+  >
+  BOOST_FORCEINLINE
+  std::size_t unprotected_bulk_visit(
+    GroupAccessMode access_mode,FwdIterator first,F&& f)const
+  {
+    std::size_t res=0,
+                hashes[M],
+                positions[M];
+    int         masks[M];
+    auto        it=first;
+
+    for(std::size_t i=0;i<M;++i,++it){
+      hashes[i]=this->hash_for(*it);
+      auto pos=positions[i]=this->position_for(hashes[i]);
+      BOOST_UNORDERED_PREFETCH(this->arrays.groups()+pos);
+    }
+
+    for(std::size_t i=0;i<M;++i){
+      auto hash=hashes[i];
+      auto pos=positions[i];
+      masks[i]=(this->arrays.groups()+pos)->match(hash);
+      if(masks[i]){
+        BOOST_UNORDERED_PREFETCH(this->arrays.group_accesses()+pos);
+        BOOST_UNORDERED_PREFETCH_ELEMENTS(this->arrays.elements()+pos*N,N);
+      }
+    }
+
+    it=first;
+    for(std::size_t i=0;i<M;++i,++it){
+      auto          pos=positions[i];
+      prober        pb(pos);
+      auto          pg=this->arrays.groups()+pos;
+      auto          mask=masks[i];
+      element_type *p;
+      if(mask){
+        p=this->arrays.elements()+pos*N;
+        goto post_prefetch;
+      }
+      else{
+        goto post_mask;
+      }
+      do{
+        pos=pb.get();
+        pg=this->arrays.groups()+pos;
+        mask=pg->match(hashes[i]);
+        if(BOOST_UNLIKELY(mask!=0)){ /* unlikely bc we're past the 1st probe */
+          p=this->arrays.elements()+pos*N;
+          BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
+      post_prefetch:
+          auto lck=access(access_mode,pos);
+          do{
+            auto n=unchecked_countr_zero(mask);
+            if(BOOST_LIKELY(
+              pg->is_occupied(n)&&
+              bool(this->pred()(*it,this->key_from(p[n]))))){
+              f(cast_for(access_mode,type_policy::value_from(p[n])));
+              ++res;
+              goto next_key;
+            }
+            mask&=mask-1;
+          }while(mask);
+        }
+      post_mask:
+        if(BOOST_LIKELY(pg->is_not_overflowed(hashes[i]))){
+          goto next_key;
+        }
+      }
+      while(BOOST_LIKELY(pb.next(this->arrays.groups_size_mask)));
+      next_key:;
+    }
+    return res;
+  }
+#else
+  template<typename GroupAccessMode,typename FwdIterator,typename F>
+  BOOST_FORCEINLINE std::size_t unprotected_bulk_visit(
+    GroupAccessMode access_mode,FwdIterator first,std::size_t m,F&& f)const
+  {
+    BOOST_ASSERT(s<2*bulk_visit_size);
+
+    std::size_t res=0,
+                hashes[2*bulk_visit_size-1],
+                positions[2*bulk_visit_size-1];
+    int         masks[2*bulk_visit_size-1];
+    auto        it=first;
+
+    for(auto i=m;i--;++it){
+      auto hash=hashes[i]=this->hash_for(*it);
+      auto pos=positions[i]=this->position_for(hash);
+      BOOST_UNORDERED_PREFETCH(this->arrays.groups()+pos);
+    }
+
+    for(auto i=m;i--;){
+      auto hash=hashes[i];
+      auto pos=positions[i];
+      auto mask=masks[i]=(this->arrays.groups()+pos)->match(hash);
+      if(mask){
+        BOOST_UNORDERED_PREFETCH(this->arrays.group_accesses()+pos);
+        BOOST_UNORDERED_PREFETCH_ELEMENTS(this->arrays.elements()+pos*N,N);
+      }
+    }
+
+    it=first;
+    for(auto i=m;i--;++it){
+      auto          pos=positions[i];
+      prober        pb(pos);
+      auto          pg=this->arrays.groups()+pos;
+      auto          mask=masks[i];
+      element_type *p;
+      if(!mask)goto post_mask;
+      p=this->arrays.elements()+pos*N;
+      for(;;){
+        {
+          auto lck=access(access_mode,pos);
+          do{
+            auto n=unchecked_countr_zero(mask);
+            if(BOOST_LIKELY(
+              pg->is_occupied(n)&&
+              bool(this->pred()(*it,this->key_from(p[n]))))){
+              f(cast_for(access_mode,type_policy::value_from(p[n])));
+              ++res;
+              goto next_key;
+            }
+            mask&=mask-1;
+          }while(mask);
+        }
+      post_mask:
+        if(BOOST_LIKELY(pg->is_not_overflowed(hashes[i]))||
+           BOOST_UNLIKELY(!pb.next(this->arrays.groups_size_mask))){
+          goto next_key;
+        }
+        pos=pb.get();
+        pg=this->arrays.groups()+pos;
+        mask=pg->match(hashes[i]);
+        if(BOOST_LIKELY(mask==0))goto next_key;
+        p=this->arrays.elements()+pos*N;
+        BOOST_UNORDERED_PREFETCH_ELEMENTS(p,N);
+      }
+      next_key:;
+    }
+    return res;
+  }
+#endif
+
 #if defined(BOOST_MSVC)
 #pragma warning(pop) /* C4800 */
 #endif