Feature/concurrent node containers (#271)

* added concurrent node containers

* removed spurious typename

* added missing includes

* avoided unused param warning

* worked around Clang bug

* s/{}/() to work around GCC4.8 problems with aggregate initialization

* used /bigobj for cfoa/visit_tests.cpp

* suppressed localized maybe-uninitialized warnings

* fixed comments

* added /bigobj to cfoa/insert_tests.cpp

* instrumented double exact comparison to spot a spurious error

* fixed pedantic error

* refactored byte_span machinery

* compromised on sub-epsilon equality for doubles that should be identical

* documented boost::concurrent_node_(map|set)

* added concurrent_node_set

* added missing AlternativeType

* tested empty node insertion

* tested node_handle allocator management

* added nonassignable_allocator and node_handle_allocator_swap_tests

* fixed warning disabling

* silenced spurious GCC warning

* broadened scope of previous pragma

* broadened even more

* worked around spurious constexpr-related msvc-14.0 bug
https://godbolt.org/z/v78545Ebf

* added workaround back

* replaced previous workaround with built-in one

* added workaround back on top of built-in solution (which doesn't work 100% of the time)

include/boost/unordered/detail/foa/concurrent_table.hpp

@@ -397,10 +397,10 @@ inline void swap(atomic_size_control& x,atomic_size_control& y)
  *   - Parallel versions of [c]visit_all(f) and erase_if(f) are provided based
  *     on C++17 stdlib parallel algorithms.
  * 
- * Consult boost::concurrent_flat_(map|set) docs for the full API reference.
- * Heterogeneous lookup is suported by default, that is, without checking for
- * any ::is_transparent typedefs --this checking is done by the wrapping
- * containers.
+ * Consult boost::concurrent_(flat|node)_(map|set) docs for the full API
+ * reference. Heterogeneous lookup is suported by default, that is, without
+ * checking for any ::is_transparent typedefs --this checking is done by the
+ * wrapping containers.
  *
  * Thread-safe concurrency is implemented using a two-level lock system:
  * 
@@ -724,6 +724,14 @@ public:
   BOOST_FORCEINLINE bool
   insert(value_type&& x){return emplace_impl(std::move(x));}
 
+  template<typename T=element_type>
+  BOOST_FORCEINLINE
+  typename std::enable_if<
+    !std::is_same<T,value_type>::value,
+    bool
+  >::type
+  insert(element_type&& x){return emplace_impl(std::move(x));}
+
   template<typename Key,typename... Args>
   BOOST_FORCEINLINE bool try_emplace(Key&& x,Args&&... args)
   {
@@ -819,6 +827,30 @@ public:
       group_shared{},std::forward<F>(f),std::move(x));
   }
 
+  template<typename F,typename T=element_type>
+  BOOST_FORCEINLINE
+  typename std::enable_if<
+    !std::is_same<T,value_type>::value,
+    bool
+  >::type
+  insert_or_visit(element_type&& x, F&& f)
+  {
+    return emplace_or_visit_impl(
+      group_exclusive{},std::forward<F>(f),std::move(x));
+  }
+
+  template<typename F,typename T=element_type>
+  BOOST_FORCEINLINE
+  typename std::enable_if<
+    !std::is_same<T,value_type>::value,
+    bool
+  >::type
+  insert_or_cvisit(element_type&& x, F&& f)
+  {
+    return emplace_or_visit_impl(
+      group_shared{},std::forward<F>(f),std::move(x));
+  }
+
   template<typename Key>
   BOOST_FORCEINLINE std::size_t erase(const Key& x)
   {
@@ -889,6 +921,29 @@ public:
     super::clear();
   }
 
+  template<typename Key,typename Extractor>
+  BOOST_FORCEINLINE void extract(const Key& x,Extractor&& ext)
+  {
+    extract_if(
+      x,[](const value_type&){return true;},std::forward<Extractor>(ext));
+  }
+
+  template<typename Key,typename F,typename Extractor>
+  BOOST_FORCEINLINE void extract_if(const Key& x,F&& f,Extractor&& ext)
+  {
+    auto        lck=shared_access();
+    auto        hash=this->hash_for(x);
+    unprotected_internal_visit(
+      group_exclusive{},x,this->position_for(hash),hash,
+      [&,this](group_type* pg,unsigned int n,element_type* p)
+      {
+        if(f(cast_for(group_exclusive{},type_policy::value_from(*p)))){
+          ext(std::move(*p),this->al());
+          super::erase(pg,n,p);
+        }
+      });
+  }
+
   // TODO: should we accept different allocator too?
   template<typename Hash2,typename Pred2>
   size_type merge(concurrent_table<TypePolicy,Hash2,Pred2,Allocator>& x)
@@ -1733,7 +1788,8 @@ private:
 
       if(this->find(x,pos0,hash))throw_exception(bad_archive_exception());
       auto loc=this->unchecked_emplace_at(pos0,hash,std::move(x));
-      ar.reset_object_address(std::addressof(*loc.p),std::addressof(x));
+      ar.reset_object_address(
+        std::addressof(type_policy::value_from(*loc.p)),std::addressof(x));
     }
   }
 
@@ -1742,7 +1798,7 @@ private:
   {
     using raw_key_type=typename std::remove_const<key_type>::type;
     using raw_mapped_type=typename std::remove_const<
-      typename TypePolicy::mapped_type>::type;
+      typename type_policy::mapped_type>::type;
 
     auto                                   lck=exclusive_access();
     std::size_t                            s;
@@ -1766,8 +1822,12 @@ private:
 
       if(this->find(k,pos0,hash))throw_exception(bad_archive_exception());
       auto loc=this->unchecked_emplace_at(pos0,hash,std::move(k),std::move(m));
-      ar.reset_object_address(std::addressof(loc.p->first),std::addressof(k));
-      ar.reset_object_address(std::addressof(loc.p->second),std::addressof(m));
+      ar.reset_object_address(
+        std::addressof(type_policy::value_from(*loc.p).first),
+        std::addressof(k));
+      ar.reset_object_address(
+        std::addressof(type_policy::value_from(*loc.p).second),
+        std::addressof(m));
     }
   }
 

include/boost/unordered/detail/foa/core.hpp

@@ -1459,6 +1459,11 @@ public:
   using stats=table_core_stats;
 #endif
 
+#if defined(BOOST_GCC)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#endif
+
   table_core(
     std::size_t n=default_bucket_count,const Hash& h_=Hash(),
     const Pred& pred_=Pred(),const Allocator& al_=Allocator()):
@@ -1467,6 +1472,10 @@ public:
     size_ctrl{initial_max_load(),0}
     {}
 
+#if defined(BOOST_GCC)
+#pragma GCC diagnostic pop
+#endif
+
   /* genericize on an ArraysFn so that we can do things like delay an
    * allocation for the group_access data required by cfoa after the move
    * constructors of Hash, Pred have been invoked
@@ -2081,6 +2090,11 @@ private:
   using pred_base=empty_value<Pred,1>;
   using allocator_base=empty_value<Allocator,2>;
 
+#if defined(BOOST_GCC)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#endif
+
   /* used by allocator-extended move ctor */
 
   table_core(Hash&& h_,Pred&& pred_,const Allocator& al_):
@@ -2091,6 +2105,10 @@ private:
   {
   }
 
+#if defined(BOOST_GCC)
+#pragma GCC diagnostic pop
+#endif
+
   arrays_type new_arrays(std::size_t n)const
   {
     return arrays_type::new_(typename arrays_type::allocator_type(al()),n);

include/boost/unordered/detail/foa/node_handle.hpp

@@ -1,4 +1,5 @@
 /* Copyright 2023 Christian Mazakas.
+ * Copyright 2024 Joaquin M Lopez Munoz.
  * Distributed under the Boost Software License, Version 1.0.
  * (See accompanying file LICENSE_1_0.txt or copy at
  * http://www.boost.org/LICENSE_1_0.txt)
@@ -11,8 +12,11 @@
 
 #include <boost/unordered/detail/opt_storage.hpp>
 
+#include <boost/assert.hpp>
 #include <boost/config.hpp>
+#include <boost/config/workaround.hpp>
 #include <boost/core/allocator_access.hpp>
+#include <type_traits>
 
 namespace boost{
 namespace unordered{
@@ -27,6 +31,13 @@ struct insert_return_type
   NodeType node;
 };
 
+template <class NodeType>
+struct iteratorless_insert_return_type
+{
+  bool     inserted;
+  NodeType node;
+};
+
 template <class TypePolicy,class Allocator>
 struct node_handle_base
 {
@@ -42,7 +53,27 @@ struct node_handle_base
     element_type p_;
     BOOST_ATTRIBUTE_NO_UNIQUE_ADDRESS opt_storage<Allocator> a_;
 
-  protected:
+    friend struct node_handle_access;
+
+    template<bool B>
+    void move_assign_allocator_if(node_handle_base&& nh)noexcept
+    {
+      move_assign_allocator_if(
+        std::integral_constant<bool,B>{}, std::move(nh));
+    }
+
+    void move_assign_allocator_if(
+      std::true_type, node_handle_base&& nh)noexcept
+    {
+      al()=std::move(nh.al());
+    }
+
+    void move_assign_allocator_if(
+      std::false_type, node_handle_base&&)noexcept
+    {
+    }
+
+protected:
     node_value_type& data()noexcept
     {
       return *(p_.p);
@@ -126,9 +157,7 @@ struct node_handle_base
             BOOST_ASSERT(pocma||al()==nh.al());
 
             type_policy::destroy(al(),&p_);
-            if(pocma){
-              al()=std::move(nh.al());
-            }
+            move_assign_allocator_if<pocma>(std::move(nh));
 
             p_=std::move(nh.p_);
             nh.reset();
@@ -153,7 +182,17 @@ struct node_handle_base
       }
     }
 
-    allocator_type get_allocator()const noexcept{return al();}
+    allocator_type get_allocator()const
+    {
+#if defined(BOOST_GCC)
+      /* GCC lifetime analysis incorrectly warns about uninitialized
+       * allocator object under some circumstances.
+       */
+      if(empty())__builtin_unreachable();
+#endif
+      return al();
+    }
+
     explicit operator bool()const noexcept{ return !empty();}
     BOOST_ATTRIBUTE_NODISCARD bool empty()const noexcept{return p_.p==nullptr;}
 
@@ -196,6 +235,82 @@ struct node_handle_base
     }
 };
 
+// Internal usage of node_handle_base protected API
+
+struct node_handle_access
+{
+  template <class TypePolicy, class Allocator>
+  using node_type = node_handle_base<TypePolicy, Allocator>;
+
+#if BOOST_WORKAROUND(BOOST_CLANG_VERSION,<190000)
+  // https://github.com/llvm/llvm-project/issues/25708
+
+  template <class TypePolicy, class Allocator>
+  struct element_type_impl
+  {
+    using type = typename node_type<TypePolicy, Allocator>::element_type;
+  };
+  template <class TypePolicy, class Allocator>
+  using element_type = typename element_type_impl<TypePolicy, Allocator>::type;
+#else
+  template <class TypePolicy, class Allocator>
+  using element_type = typename node_type<TypePolicy, Allocator>::element_type;
+#endif
+
+  template <class TypePolicy, class Allocator>
+  static element_type<TypePolicy, Allocator>&
+  element(node_type<TypePolicy, Allocator>& nh)noexcept
+  {
+    return nh.element();
+  }
+
+  template <class TypePolicy, class Allocator>
+  static element_type<TypePolicy, Allocator>
+  const& element(node_type<TypePolicy, Allocator> const& nh)noexcept
+  {
+    return nh.element();
+  }
+
+  template <class TypePolicy, class Allocator>
+  static void emplace(
+    node_type<TypePolicy, Allocator>& nh,
+    element_type<TypePolicy, Allocator>&& x, Allocator a)
+  {
+    nh.emplace(std::move(x), a);
+  }
+
+  template <class TypePolicy,class Allocator>
+  static void reset(node_type<TypePolicy, Allocator>& nh)
+  {
+    nh.reset();
+  }
+};
+
+template <class TypePolicy, class Allocator>
+class node_handle_emplacer_class
+{
+  using access = node_handle_access;
+  using node_type = access::node_type<TypePolicy, Allocator>;
+  using element_type = access::element_type<TypePolicy, Allocator>;
+
+  node_type & nh;
+
+public:
+  node_handle_emplacer_class(node_type& nh_): nh(nh_) {}
+
+  void operator()(element_type&& x,Allocator a)
+  {
+    access::emplace(nh, std::move(x), a);
+  }
+};
+
+template <class TypePolicy, class Allocator>
+node_handle_emplacer_class<TypePolicy, Allocator>
+node_handle_emplacer(node_handle_base<TypePolicy, Allocator>& nh)
+{
+  return {nh};
+}
+
 }
 }
 }

include/boost/unordered/detail/foa/node_map_handle.hpp

@@ -0,0 +1,56 @@
+/* Copyright 2023 Christian Mazakas.
+ * Copyright 2024 Joaquin M Lopez Munoz. 
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_HANDLE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_HANDLE_HPP
+
+#include <boost/unordered/detail/foa/node_handle.hpp>
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+template <class TypePolicy, class Allocator>
+struct node_map_handle
+    : public node_handle_base<TypePolicy, Allocator>
+{
+private:
+  using base_type = node_handle_base<TypePolicy, Allocator>;
+
+  using typename base_type::type_policy;
+
+public:
+  using key_type = typename TypePolicy::key_type;
+  using mapped_type = typename TypePolicy::mapped_type;
+
+  constexpr node_map_handle() noexcept = default;
+  node_map_handle(node_map_handle&& nh) noexcept = default;
+
+  node_map_handle& operator=(node_map_handle&&) noexcept = default;
+
+  key_type& key() const
+  {
+    BOOST_ASSERT(!this->empty());
+    return const_cast<key_type&>(this->data().first);
+  }
+
+  mapped_type& mapped() const
+  {
+    BOOST_ASSERT(!this->empty());
+    return const_cast<mapped_type&>(this->data().second);
+  }
+};
+
+}
+}
+}
+}
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_NODE_MAP_HANDLE_HPP

include/boost/unordered/detail/foa/node_set_handle.hpp

@@ -0,0 +1,48 @@
+/* Copyright 2023 Christian Mazakas.
+ * Copyright 2024 Joaquin M Lopez Munoz. 
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_NODE_SET_HANDLE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_NODE_SET_HANDLE_HPP
+
+#include <boost/unordered/detail/foa/node_handle.hpp>
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+template <class TypePolicy, class Allocator>
+struct node_set_handle
+    : public detail::foa::node_handle_base<TypePolicy, Allocator>
+{
+private:
+  using base_type = detail::foa::node_handle_base<TypePolicy, Allocator>;
+
+  using typename base_type::type_policy;
+
+public:
+  using value_type = typename TypePolicy::value_type;
+
+  constexpr node_set_handle() noexcept = default;
+  node_set_handle(node_set_handle&& nh) noexcept = default;
+  node_set_handle& operator=(node_set_handle&&) noexcept = default;
+
+  value_type& value() const
+  {
+    BOOST_ASSERT(!this->empty());
+    return const_cast<value_type&>(this->data());
+  }
+};
+
+}
+}
+}
+}
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_NODE_SET_HANDLE_HPP