Refactor node_handle to directly store element_type by modularizing and extending it

include/boost/unordered/detail/foa/element_type.hpp

@@ -0,0 +1,60 @@
+/* Copyright 2023 Christian Mazakas.
+ * Distributed under the Boost Software License, Version 1.0.
+ * (See accompanying file LICENSE_1_0.txt or copy at
+ * http://www.boost.org/LICENSE_1_0.txt)
+ *
+ * See https://www.boost.org/libs/unordered for library home page.
+ */
+
+#ifndef BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP
+#define BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP
+
+namespace boost{
+namespace unordered{
+namespace detail{
+namespace foa{
+
+template<class T>
+struct element_type
+{
+  using value_type=T;
+  value_type* p;
+
+  /*
+   * we use a deleted copy constructor here so the type is no longer
+   * trivially copy-constructible which inhibits our memcpy
+   * optimizations when copying the tables
+   */
+  element_type() = default;
+  element_type(value_type* p_):p(p_){}
+  element_type(element_type const&) = delete;
+  element_type(element_type&& rhs) noexcept
+  {
+    p = rhs.p;
+    rhs.p = nullptr;
+  }
+
+  element_type& operator=(element_type const&)=delete;
+  element_type& operator=(element_type&& rhs)noexcept
+  {
+    if (this!=&rhs){
+      p=rhs.p;
+      rhs.p=nullptr;
+    }
+    return *this;
+  }
+
+  void swap(element_type& rhs)noexcept
+  {
+    auto tmp=p;
+    p=rhs.p;
+    rhs.p=tmp;
+  }
+};
+
+}
+}
+}
+}
+
+#endif // BOOST_UNORDERED_DETAIL_FOA_ELEMENT_TYPE_HPP

include/boost/unordered/detail/foa/node_handle.hpp

@@ -45,20 +45,30 @@ struct node_handle_base
 
   private:
     using node_value_type=typename type_policy::value_type;
-    node_value_type* p_=nullptr;
+    element_type p_;
     BOOST_ATTRIBUTE_NO_UNIQUE_ADDRESS opt_storage<Allocator> a_;
 
   protected:
-    node_value_type& element()noexcept
+    node_value_type& data()noexcept
     {
-      BOOST_ASSERT(!empty());
+      return *(p_.p);
-      return *p_;
     }
 
-    node_value_type const& element()const noexcept
+    node_value_type const& data()const noexcept
+    {
+      return *(p_.p);
+    }
+
+    element_type& element()noexcept
     {
       BOOST_ASSERT(!empty());
-      return *p_;
+      return p_;
+    }
+
+    element_type const& element()const noexcept
+    {
+      BOOST_ASSERT(!empty());
+      return p_;
     }
 
     Allocator& al()noexcept
@@ -73,52 +83,46 @@ struct node_handle_base
       return a_.t_;
     }
 
-    void emplace(node_value_type* p,Allocator a)
-    {
-      BOOST_ASSERT(empty());
-      p_=p;
-      new(&a_.t_)Allocator(a);
-    }
-
     void emplace(element_type&& x,Allocator a)
     {
-      emplace(x.p,a);
+      BOOST_ASSERT(empty());
+      auto* p=x.p;
+      p_.p=p;
+      new(&a_.t_)Allocator(a);
       x.p=nullptr;
     }
 
     void reset()
     {
-      al().~Allocator();
+      a_.t_.~Allocator();
-      p_=nullptr;
+      p_.p=nullptr;
     }
 
   public:
-    constexpr node_handle_base()noexcept{}
+    constexpr node_handle_base()noexcept:p_{nullptr}{}
 
     node_handle_base(node_handle_base&& nh) noexcept
     {
+      p_.p = nullptr;
       if (!nh.empty()){
-        emplace(nh.p_,nh.al());
+        emplace(std::move(nh.p_),nh.al());
         nh.reset();
       }
     }
 
     node_handle_base& operator=(node_handle_base&& nh)noexcept
     {
-      element_type x;
-      x.p=p_;
-
       if(this!=&nh){
         if(empty()){
           if(nh.empty()){                      /* empty(),  nh.empty() */
             /* nothing to do */
           }else{                               /* empty(), !nh.empty() */
-            emplace(nh.p_,std::move(nh.al()));
+            emplace(std::move(nh.p_),std::move(nh.al()));
             nh.reset();
           }
         }else{
           if(nh.empty()){                      /* !empty(),  nh.empty() */
-            type_policy::destroy(al(),&x);
+            type_policy::destroy(al(),&p_);
             reset();
           }else{                               /* !empty(), !nh.empty() */
             bool const pocma=
@@ -127,12 +131,12 @@ struct node_handle_base
 
             BOOST_ASSERT(pocma||al()==nh.al());
 
-            type_policy::destroy(al(),&x);
+            type_policy::destroy(al(),&p_);
             if(pocma){
               al()=std::move(nh.al());
             }
 
-            p_=nh.p_;
+            p_=std::move(nh.p_);
             nh.reset();
           }
         }
@@ -140,7 +144,7 @@ struct node_handle_base
         if(empty()){                           /* empty(),  nh.empty() */
           /* nothing to do */
         }else{                                 /* !empty(), !nh.empty() */
-          type_policy::destroy(al(),&x);
+          type_policy::destroy(al(),&p_);
           reset();
         }
       }
@@ -150,16 +154,14 @@ struct node_handle_base
     ~node_handle_base()
     {
       if(!empty()){
-        element_type x;
+        type_policy::destroy(al(),&p_);
-        x.p=p_;
-        type_policy::destroy(al(),&x);
         reset();
       }
     }
 
     allocator_type get_allocator()const noexcept{return al();}
     explicit operator bool()const noexcept{ return !empty();}
-    BOOST_ATTRIBUTE_NODISCARD bool empty()const noexcept{return p_==nullptr;}
+    BOOST_ATTRIBUTE_NODISCARD bool empty()const noexcept{return p_.p==nullptr;}
 
     void swap(node_handle_base& nh) noexcept(
       boost::allocator_is_always_equal<Allocator>::type::value||
@@ -170,12 +172,12 @@ struct node_handle_base
           if(nh.empty()) {
             /* nothing to do here */
           } else {
-            emplace(nh.p_, nh.al());
+            emplace(std::move(nh.p_), nh.al());
             nh.reset();
           }
         }else{
           if(nh.empty()){
-            nh.emplace(p_,al());
+            nh.emplace(std::move(p_),al());
             reset();
           }else{
             bool const pocs=
@@ -185,7 +187,7 @@ struct node_handle_base
             BOOST_ASSERT(pocs || al()==nh.al());
 
             using std::swap;
-            swap(p_,nh.p_);
+            p_.swap(nh.p_);
             if(pocs)swap(al(),nh.al());
           }
         }

include/boost/unordered/unordered_node_map.hpp

@@ -11,6 +11,7 @@
 #endif
 
 #include <boost/unordered/detail/foa.hpp>
+#include <boost/unordered/detail/foa/element_type.hpp>
 #include <boost/unordered/detail/foa/node_handle.hpp>
 #include <boost/unordered/detail/type_traits.hpp>
 #include <boost/unordered/unordered_node_map_fwd.hpp>
@@ -45,23 +46,7 @@ namespace boost {
         using value_type = std::pair<Key const, T>;
         using moved_type = std::pair<raw_key_type&&, raw_mapped_type&&>;
 
-        struct element_type
+        using element_type=foa::element_type<value_type>;
-        {
-          value_type* p;
-
-          /*
-           * we use a deleted copy constructor here so the type is no longer
-           * trivially copy-constructible which inhibits our memcpy
-           * optimizations when copying the tables
-           */
-          element_type() = default;
-          element_type(element_type const&) = delete;
-          element_type(element_type&& rhs) noexcept
-          {
-            p = rhs.p;
-            rhs.p = nullptr;
-          }
-        };
 
         static value_type& value_from(element_type const& x) { return *(x.p); }
 
@@ -180,13 +165,13 @@ namespace boost {
         key_type& key() const
         {
           BOOST_ASSERT(!this->empty());
-          return const_cast<key_type&>(this->element().first);
+          return const_cast<key_type&>(this->data().first);
         }
 
         mapped_type& mapped() const
         {
           BOOST_ASSERT(!this->empty());
-          return const_cast<mapped_type&>(this->element().second);
+          return const_cast<mapped_type&>(this->data().second);
         }
       };
     } // namespace detail
@@ -425,10 +410,7 @@ namespace boost {
 
         BOOST_ASSERT(get_allocator() == nh.get_allocator());
 
-        typename map_types::element_type x;
+        auto itp = table_.insert(std::move(nh.element()));
-        x.p = std::addressof(nh.element());
-
-        auto itp = table_.insert(std::move(x));
         if (itp.second) {
           nh.reset();
           return {itp.first, true, node_type{}};
@@ -445,10 +427,7 @@ namespace boost {
 
         BOOST_ASSERT(get_allocator() == nh.get_allocator());
 
-        typename map_types::element_type x;
+        auto itp = table_.insert(std::move(nh.element()));
-        x.p = std::addressof(nh.element());
-
-        auto itp = table_.insert(std::move(x));
         if (itp.second) {
           nh.reset();
           return itp.first;

include/boost/unordered/unordered_node_set.hpp

@@ -11,6 +11,7 @@
 #endif
 
 #include <boost/unordered/detail/foa.hpp>
+#include <boost/unordered/detail/foa/element_type.hpp>
 #include <boost/unordered/detail/foa/node_handle.hpp>
 #include <boost/unordered/detail/type_traits.hpp>
 #include <boost/unordered/unordered_node_set_fwd.hpp>
@@ -41,23 +42,7 @@ namespace boost {
 
         static Key const& extract(value_type const& key) { return key; }
 
-        struct element_type
+        using element_type=foa::element_type<value_type>;
-        {
-          value_type* p;
-
-          /*
-           * we use a deleted copy constructor here so the type is no longer
-           * trivially copy-constructible which inhibits our memcpy
-           * optimizations when copying the tables
-           */
-          element_type() = default;
-          element_type(element_type const&) = delete;
-          element_type(element_type&& rhs) noexcept
-          {
-            p = rhs.p;
-            rhs.p = nullptr;
-          }
-        };
 
         static value_type& value_from(element_type const& x) { return *x.p; }
         static Key const& extract(element_type const& k) { return *k.p; }
@@ -142,7 +127,7 @@ namespace boost {
         value_type& value() const
         {
           BOOST_ASSERT(!this->empty());
-          return const_cast<value_type&>(this->element());
+          return const_cast<value_type&>(this->data());
         }
       };
     } // namespace detail
@@ -395,10 +380,7 @@ namespace boost {
 
         BOOST_ASSERT(get_allocator() == nh.get_allocator());
 
-        typename set_types::element_type x;
+        auto itp = table_.insert(std::move(nh.element()));
-        x.p = std::addressof(nh.element());
-
-        auto itp = table_.insert(std::move(x));
         if (itp.second) {
           nh.reset();
           return {itp.first, true, node_type{}};
@@ -415,10 +397,7 @@ namespace boost {
 
         BOOST_ASSERT(get_allocator() == nh.get_allocator());
 
-        typename set_types::element_type x;
+        auto itp = table_.insert(std::move(nh.element()));
-        x.p = std::addressof(nh.element());
-
-        auto itp = table_.insert(std::move(x));
         if (itp.second) {
           nh.reset();
           return itp.first;