protected visitation against spurious insertion sentinels

* Create function `detail::throw_out_of_range()` to make `at()` more inlineable

* Replace `boost::throw_exception()` with `detail::throw_out_of_range()`

include/boost/unordered/detail/foa/concurrent_table.hpp

@@ -198,13 +198,13 @@ private:
 template<typename Integral>
 struct atomic_integral
 {
-  operator Integral()const{return n.load(std::memory_order_relaxed);}
-  void operator=(Integral m){n.store(m,std::memory_order_relaxed);}
-  void operator|=(Integral m){n.fetch_or(m,std::memory_order_relaxed);}
-  void operator&=(Integral m){n.fetch_and(m,std::memory_order_relaxed);}
+  operator Integral()const{return n.load(std::memory_order_acquire);}
+  void operator=(Integral m){n.store(m,std::memory_order_release);}
+  void operator|=(Integral m){n.fetch_or(m);}
+  void operator&=(Integral m){n.fetch_and(m);}
 
   atomic_integral& operator=(atomic_integral const& rhs) {
-    n.store(rhs.n.load(std::memory_order_relaxed),std::memory_order_relaxed);
+    n.store(rhs.n.load());
     return *this;
   }
 
@@ -1177,7 +1177,7 @@ private:
         do{
           auto n=unchecked_countr_zero(mask);
           if(BOOST_LIKELY(
-            pg->is_occupied(n)&&bool(this->pred()(x,this->key_from(p[n]))))){
+            is_occupied(pg,n)&&bool(this->pred()(x,this->key_from(p[n]))))){
             f(pg,n,p+n);
             return 1;
           }
@@ -1236,7 +1236,7 @@ private:
           do{
             auto n=unchecked_countr_zero(mask);
             if(BOOST_LIKELY(
-              pg->is_occupied(n)&&
+              is_occupied(pg,n)&&
               bool(this->pred()(*it,this->key_from(p[n]))))){
               f(cast_for(access_mode,type_policy::value_from(p[n])));
               ++res;
@@ -1436,10 +1436,103 @@ private:
     bool        commit_=false;
   };
 
+  struct latch_free_reserve_slot
+  {
+    latch_free_reserve_slot(group_type* pg_,std::size_t pos_):pg{pg_},pos{pos_}
+    {
+      unsigned char expected=0;
+      succeeded_=reinterpret_cast<std::atomic<unsigned char>*>(pg)[pos].
+        compare_exchange_weak(expected,1);
+    }
+
+    ~latch_free_reserve_slot()
+    {
+      if(succeeded_&&!commit_)pg->reset(pos);
+    }
+
+    bool succeeded()const{return succeeded_;}
+
+    void commit(){commit_=true;}
+
+    group_type  *pg;
+    std::size_t pos;
+    bool        succeeded_;
+    bool        commit_=false;
+  };
+
+  struct assign_insert_counter_on_exit
+  {
+    ~assign_insert_counter_on_exit(){counter=x;}
+
+    group_insert_counter_type &counter;
+    boost::uint32_t            x;
+  };
+
   template<typename GroupAccessMode,typename F,typename... Args>
   BOOST_FORCEINLINE int
   unprotected_norehash_emplace_or_visit(
     GroupAccessMode access_mode,F&& f,Args&&... args)
+  {
+    return unprotected_norehash_emplace_or_visit_with_layout(
+      std::integral_constant<bool,group_type::regular_layout>{},
+      access_mode,std::forward<F>(f),std::forward<Args>(args)...);
+  }
+
+  template<typename GroupAccessMode,typename F,typename... Args>
+  BOOST_FORCEINLINE int
+  unprotected_norehash_emplace_or_visit_with_layout(
+    std::true_type, /* regular layout, almost-latch-free */
+    GroupAccessMode access_mode,F&& f,Args&&... args)
+  {
+    const auto &k=this->key_from(std::forward<Args>(args)...);
+    auto        hash=this->hash_for(k);
+    auto        pos0=this->position_for(hash);
+
+    for(;;){
+    startover:
+      boost::uint32_t counter=0;
+      while(BOOST_UNLIKELY((counter=insert_counter(pos0))%2==1)){}
+      if(unprotected_visit(
+        access_mode,k,pos0,hash,std::forward<F>(f)))return 0;
+
+      reserve_size rsize(*this);
+      if(BOOST_LIKELY(rsize.succeeded())){
+        for(prober pb(pos0);;pb.next(this->arrays.groups_size_mask)){
+          auto pos=pb.get();
+          auto pg=this->arrays.groups()+pos;
+          auto mask=pg->match_available();
+          if(BOOST_LIKELY(mask!=0)){
+            auto n=unchecked_countr_zero(mask);
+            latch_free_reserve_slot rslot{pg,n};
+            if(BOOST_UNLIKELY(!rslot.succeeded())){
+              /* slot wasn't empty */
+              goto startover;
+            }
+            if(BOOST_UNLIKELY(
+              !insert_counter(pos0).compare_exchange_weak(counter,counter+1))){
+              /* other thread inserted from pos0, need to start over */
+              goto startover;
+            }
+            assign_insert_counter_on_exit a{insert_counter(pos0),counter+2};
+            auto p=this->arrays.elements()+pos*N+n;
+            this->construct_element(p,std::forward<Args>(args)...);
+            pg->set(n,hash);
+            rslot.commit();
+            rsize.commit();
+            return 1;
+          }
+          pg->mark_overflow(hash);
+        }
+      }
+      else return -1;
+    }
+  }
+
+  template<typename GroupAccessMode,typename F,typename... Args>
+  BOOST_FORCEINLINE int
+  unprotected_norehash_emplace_or_visit_with_layout(
+    std::false_type, /* non-regular layout, latched */
+    GroupAccessMode access_mode,F&& f,Args&&... args)
   {
     const auto &k=this->key_from(std::forward<Args>(args)...);
     auto        hash=this->hash_for(k);
@@ -1486,6 +1579,46 @@ private:
     }
   }
 
+  /* check occupation with previous unsynced match */
+
+  static bool is_occupied(group_type* pg,std::size_t pos)
+  {
+    return is_occupied(
+      std::integral_constant<bool,group_type::regular_layout>{},pg,pos);
+  }
+
+  static bool is_occupied(std::true_type,group_type* pg,std::size_t pos)
+  {
+    /* regular layout, almost-latch-free insertion -> possible reserved slot */
+    return reinterpret_cast<std::atomic<unsigned char>*>(pg)[pos]>1;
+  }
+
+  static bool is_occupied(std::false_type,group_type* pg,std::size_t pos)
+  {
+    /* non-regular layout, latched insertion -> no reserved slots */
+    return pg->is_occupied(pos);
+  }
+
+  /* check occupation with previous synced match */
+
+  static bool is_really_occupied(group_type* pg,std::size_t pos)
+  {
+    return is_really_occupied(
+      std::integral_constant<bool,group_type::regular_layout>{},pg,pos);
+  }
+
+  static bool is_really_occupied(std::true_type,group_type* pg,std::size_t pos)
+  {
+    /* regular layout, almost-latch-free insertion -> possible reserved slot */
+    return reinterpret_cast<std::atomic<unsigned char>*>(pg)[pos]>1;
+  }
+
+  static bool is_really_occupied(std::false_type,group_type*,std::size_t)
+  {
+    /* non-regular layout, latched insertion -> no false positives */
+    return true;
+  }
+
   template<typename GroupAccessMode,typename F>
   auto for_all_elements(GroupAccessMode access_mode,F f)const
     ->decltype(f(nullptr),void())
@@ -1523,7 +1656,7 @@ private:
         auto mask=this->match_really_occupied(pg,last);
         while(mask){
           auto n=unchecked_countr_zero(mask);
-          if(!f(pg,n,p+n))return false;
+          if(BOOST_LIKELY(is_really_occupied(pg,n))&&!f(pg,n,p+n))return false;
           mask&=mask-1;
         }
       }
@@ -1558,7 +1691,7 @@ private:
         auto mask=this->match_really_occupied(&g,last);
         while(mask){
           auto n=unchecked_countr_zero(mask);
-          f(&g,n,p+n);
+          if(BOOST_LIKELY(is_really_occupied(&g,n)))f(&g,n,p+n);
           mask&=mask-1;
         }
       }
@@ -1580,7 +1713,7 @@ private:
         auto mask=this->match_really_occupied(&g,last);
         while(mask){
           auto n=unchecked_countr_zero(mask);
-          if(!f(p+n))return false;
+          if(BOOST_LIKELY(is_really_occupied(&g,n))&&!f(p+n))return false;
           mask&=mask-1;
         }
         return true;

include/boost/unordered/detail/throw_exception.hpp

@@ -0,0 +1,30 @@
+// Copyright (C) 2023 Braden Ganetsky
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_UNORDERED_DETAIL_THROW_EXCEPTION_HPP
+#define BOOST_UNORDERED_DETAIL_THROW_EXCEPTION_HPP
+
+#include <boost/config.hpp>
+#if defined(BOOST_HAS_PRAGMA_ONCE)
+#pragma once
+#endif
+
+#include <boost/throw_exception.hpp>
+#include <stdexcept>
+
+namespace boost {
+  namespace unordered {
+    namespace detail {
+
+      BOOST_NOINLINE BOOST_NORETURN inline void throw_out_of_range(
+        char const* message)
+      {
+        boost::throw_exception(std::out_of_range(message));
+      }
+
+    } // namespace detail
+  } // namespace unordered
+} // namespace boost
+
+#endif // BOOST_UNORDERED_DETAIL_THROW_EXCEPTION_HPP

include/boost/unordered/unordered_flat_map.hpp

@@ -14,12 +14,12 @@
 #include <boost/unordered/detail/foa/flat_map_types.hpp>
 #include <boost/unordered/detail/foa/table.hpp>
 #include <boost/unordered/detail/serialize_container.hpp>
+#include <boost/unordered/detail/throw_exception.hpp>
 #include <boost/unordered/detail/type_traits.hpp>
 #include <boost/unordered/unordered_flat_map_fwd.hpp>
 
 #include <boost/core/allocator_access.hpp>
 #include <boost/container_hash/hash.hpp>
-#include <boost/throw_exception.hpp>
 
 #include <initializer_list>
 #include <iterator>
@@ -459,8 +459,8 @@ namespace boost {
         // TODO: someday refactor this to conditionally serialize the key and
         // include it in the error message
         //
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_flat_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_flat_map");
       }
 
       mapped_type const& at(key_type const& key) const
@@ -469,8 +469,8 @@ namespace boost {
         if (pos != table_.end()) {
           return pos->second;
         }
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_flat_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_flat_map");
       }
 
       template <class K>
@@ -483,8 +483,8 @@ namespace boost {
         if (pos != table_.end()) {
           return pos->second;
         }
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_flat_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_flat_map");
       }
 
       template <class K>
@@ -497,8 +497,8 @@ namespace boost {
         if (pos != table_.end()) {
           return pos->second;
         }
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_flat_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_flat_map");
       }
 
       BOOST_FORCEINLINE mapped_type& operator[](key_type const& key)

include/boost/unordered/unordered_map.hpp

@@ -16,6 +16,7 @@
 
 #include <boost/unordered/detail/map.hpp>
 #include <boost/unordered/detail/serialize_fca_container.hpp>
+#include <boost/unordered/detail/throw_exception.hpp>
 #include <boost/unordered/detail/type_traits.hpp>
 
 #include <boost/container_hash/hash.hpp>
@@ -1586,8 +1587,8 @@ namespace boost {
           return p->value().second;
       }
 
-      boost::throw_exception(
-        std::out_of_range("Unable to find key in unordered_map."));
+      boost::unordered::detail::throw_out_of_range(
+        "Unable to find key in unordered_map.");
     }
 
     template <class K, class T, class H, class P, class A>
@@ -1602,8 +1603,8 @@ namespace boost {
           return p->value().second;
       }
 
-      boost::throw_exception(
-        std::out_of_range("Unable to find key in unordered_map."));
+      boost::unordered::detail::throw_out_of_range(
+        "Unable to find key in unordered_map.");
     }
 
     template <class K, class T, class H, class P, class A>
@@ -1620,8 +1621,8 @@ namespace boost {
           return p->value().second;
       }
 
-      boost::throw_exception(
-        std::out_of_range("Unable to find key in unordered_map."));
+      boost::unordered::detail::throw_out_of_range(
+        "Unable to find key in unordered_map.");
     }
 
     template <class K, class T, class H, class P, class A>
@@ -1638,8 +1639,8 @@ namespace boost {
           return p->value().second;
       }
 
-      boost::throw_exception(
-        std::out_of_range("Unable to find key in unordered_map."));
+      boost::unordered::detail::throw_out_of_range(
+        "Unable to find key in unordered_map.");
     }
 
     template <class K, class T, class H, class P, class A>

include/boost/unordered/unordered_node_map.hpp

@@ -15,12 +15,12 @@
 #include <boost/unordered/detail/foa/node_map_types.hpp>
 #include <boost/unordered/detail/foa/table.hpp>
 #include <boost/unordered/detail/serialize_container.hpp>
+#include <boost/unordered/detail/throw_exception.hpp>
 #include <boost/unordered/detail/type_traits.hpp>
 #include <boost/unordered/unordered_node_map_fwd.hpp>
 
 #include <boost/core/allocator_access.hpp>
 #include <boost/container_hash/hash.hpp>
-#include <boost/throw_exception.hpp>
 
 #include <initializer_list>
 #include <iterator>
@@ -554,8 +554,8 @@ namespace boost {
         // TODO: someday refactor this to conditionally serialize the key and
         // include it in the error message
         //
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_node_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_node_map");
       }
 
       mapped_type const& at(key_type const& key) const
@@ -564,8 +564,8 @@ namespace boost {
         if (pos != table_.end()) {
           return pos->second;
         }
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_node_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_node_map");
       }
 
       template <class K>
@@ -578,8 +578,8 @@ namespace boost {
         if (pos != table_.end()) {
           return pos->second;
         }
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_node_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_node_map");
       }
 
       template <class K>
@@ -592,8 +592,8 @@ namespace boost {
         if (pos != table_.end()) {
           return pos->second;
         }
-        boost::throw_exception(
-          std::out_of_range("key was not found in unordered_node_map"));
+        boost::unordered::detail::throw_out_of_range(
+          "key was not found in unordered_node_map");
       }
 
       BOOST_FORCEINLINE mapped_type& operator[](key_type const& key)