From 30af471e3c8c8bc97a19f428cbab3d853b8eaa55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ion=20Gazta=C3=B1aga?= Date: Sat, 1 Aug 2026 16:31:05 +0200 Subject: [PATCH] Manualy inline and optimize algorithms using the cleanup_blocks strategy. --- .../experimental/segmented_copy_if.hpp | 76 +++++----- .../experimental/segmented_partition_copy.hpp | 130 +++++++++--------- .../experimental/segmented_remove_copy.hpp | 71 +++++----- .../experimental/segmented_remove_copy_if.hpp | 73 +++++----- 4 files changed, 191 insertions(+), 159 deletions(-) diff --git a/include/boost/container/experimental/segmented_copy_if.hpp b/include/boost/container/experimental/segmented_copy_if.hpp index dae7bda..23db367 100644 --- a/include/boost/container/experimental/segmented_copy_if.hpp +++ b/include/boost/container/experimental/segmented_copy_if.hpp @@ -58,7 +58,7 @@ segmented_copy_if_dst_bounded //element. With an unreachable_sentinel_t destination both checks fold away, //so the flat path is unchanged. if(BOOST_UNLIKELY(dst_first == dst_last)) - return segduo(first, dst_first); + goto out_path; BOOST_CONTAINER_SEGMENTED_UNROLL(4) for(; first != last; ++first) { @@ -75,46 +75,58 @@ segmented_copy_if_dst_bounded return segduo(first, dst_first); } -template -BOOST_CONTAINER_FORCEINLINE segtrio -copy_if_cleanup_blocks - (RASrcIter cur, RADstIter dst_cur, RADstIter dst_last, - Pred pred, Diff avail) -{ - const Diff block_size = static_cast(BlockSize); - while(avail >= block_size && - static_cast(dst_last - dst_cur) >= block_size) { - avail -= block_size; - BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL - for(Diff chunk = block_size; chunk; ) { - --chunk; - if(pred(*cur)) { - *dst_cur = *cur; - ++dst_cur; - } - ++cur; - } - } - return segtrio(cur, dst_cur, avail); -} - template BOOST_CONTAINER_FORCEINLINE typename iterator_enable_if_tag >::type segmented_copy_if_dst_bounded (RASrcIter first, RASrcIter last, RADstIter dst_first, RADstIter dst_last, Pred pred, - const non_segmented_iterator_tag &, const std::random_access_iterator_tag &src_tag) + const non_segmented_iterator_tag &, const std::random_access_iterator_tag &) { + //[alg.copy] same dest-after-write contract as the generic leaf. Dest capacity + //is tested only after a write, so a miss scan never touches the destination + //bound. The old blocked helper rechecked (dst_last - dst) every 32 source + //elements even when nothing was written. typedef typename iterator_traits::difference_type difference_type; + typedef segduo result_t; + const difference_type block_size = 16; + difference_type n = last - first; - (void)src_tag; - segtrio r = - (copy_if_cleanup_blocks<32>) - (first, dst_first, dst_last, pred, last - first); + if(BOOST_UNLIKELY(dst_first == dst_last)) + goto out_ret; - return (segmented_copy_if_dst_bounded) - (r.first, last, r.second, dst_last, pred, non_segmented_iterator_tag(), int()); + // Avoid destination check if both input and output ranges are big enough + while(n >= block_size && + static_cast(dst_last - dst_first) >= block_size) { + n -= block_size; + BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL + for(difference_type chunk = block_size; chunk; ) { + --chunk; + if(pred(*first)) { + *dst_first = *first; + ++dst_first; + } + ++first; + } + } + + //Loop 1 may exhaust the destination exactly; do not write past the end. + if(BOOST_UNLIKELY(dst_first == dst_last)) + goto out_ret; + + // Remaining elements + BOOST_CONTAINER_SEGMENTED_UNROLL(4) + for(; n; --n, ++first) { + if(pred(*first)) { + *dst_first = *first; + ++dst_first; + if(BOOST_UNLIKELY(dst_first == dst_last)) { + ++first; + goto out_ret; + } + } + } + out_ret: + return result_t(first, dst_first); } template diff --git a/include/boost/container/experimental/segmented_partition_copy.hpp b/include/boost/container/experimental/segmented_partition_copy.hpp index d09a1f9..8cd7618 100644 --- a/include/boost/container/experimental/segmented_partition_copy.hpp +++ b/include/boost/container/experimental/segmented_partition_copy.hpp @@ -68,46 +68,6 @@ BOOST_CONTAINER_FORCEINLINE bool partition_copy_room_enough (TIter, unreachable_sentinel_t, FIter, unreachable_sentinel_t, Diff avail) { return avail >= static_cast(BlockSize); } -template -BOOST_CONTAINER_FORCEINLINE segquartet -partition_copy_cleanup_blocks - (RASrcIter cur, TIter t_first, TSent t_last, FIter f_first, FSent f_last, - Pred pred, Diff avail, dtl::true_type) -{ - const Diff block_size = static_cast(BlockSize); - while((partition_copy_room_enough) - (t_first, t_last, f_first, f_last, avail)) { - avail -= block_size; - BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL - for(Diff chunk = block_size; chunk; ) { - --chunk; - if(pred(*cur)) { - *t_first = *cur; - ++t_first; - } - else { - *f_first = *cur; - ++f_first; - } - ++cur; - } - } - return segquartet - (cur, t_first, f_first, avail); -} - -template -BOOST_CONTAINER_FORCEINLINE segquartet -partition_copy_cleanup_blocks - (RASrcIter cur, TIter t_first, TSent, FIter f_first, FSent, Pred, Diff avail, - dtl::false_type) -{ - return segquartet - (cur, t_first, f_first, avail); -} - template struct pc_output_is_ra { @@ -140,10 +100,12 @@ partition_copy_leaf // untested element, and the loop body can take for granted that both outputs // have room. With unreachable_sentinel_t outputs the tests fold away, so the // flat path is unchanged. + typedef segquartet result_t; + if(BOOST_UNLIKELY(t_first == t_last)) - return segquartet(first, t_first, f_first, false); + return result_t(first, t_first, f_first, false); if(BOOST_UNLIKELY(f_first == f_last)) - return segquartet(first, t_first, f_first, first != last); + return result_t(first, t_first, f_first, first != last); bool false_output_full = false; BOOST_CONTAINER_SEGMENTED_UNROLL(4) @@ -166,15 +128,13 @@ partition_copy_leaf } } } - return segquartet - (first, t_first, f_first, false_output_full); + return result_t(first, t_first, f_first, false_output_full); } -// Random-access-source fast path (needs random-access outputs too). Process -// fixed 32-element source blocks while both outputs have room for the worst -// case (all 32 elements routed to either output), use 8-element cleanup blocks -// when an output is bounded, then finish with the generic checked loop. An -// unbounded output reports the available source count as its room. +// Random-access-source fast path (needs random-access outputs too). +// Three loops: both outputs have room for a full block; source-only blocks +// with per-write checks; residual. An unbounded output folds its capacity +// tests away via unreachable_sentinel_t. template BOOST_CONTAINER_FORCEINLINE typename algo_enable_if_c @@ -185,24 +145,66 @@ partition_copy_leaf Pred pred, const std::random_access_iterator_tag &) { typedef typename iterator_traits::difference_type difference_type; - typedef segquartet cleanup_result; + typedef segquartet result_t; + const difference_type block_size = 16; + difference_type n = last - first; + bool false_output_full = false; - const cleanup_result r32 = (partition_copy_cleanup_blocks<32>) - (first, t_first, t_last, f_first, f_last, pred, last - first, - dtl::true_type()); + if(BOOST_UNLIKELY(t_first == t_last)) + return result_t(first, t_first, f_first, false); + if(BOOST_UNLIKELY(f_first == f_last)) + return result_t(first, t_first, f_first, first != last); - typedef dtl::integral_constant - < bool - , !dtl::is_same::value || - !dtl::is_same::value - > has_bounded_output_t; - const cleanup_result r8 = (partition_copy_cleanup_blocks<8>) - (r32.first, r32.second, t_last, r32.third, f_last, pred, r32.fourth, - has_bounded_output_t()); + //Avoid output checks if source and both outputs are big enough + while((partition_copy_room_enough<16>) + (t_first, t_last, f_first, f_last, n)) { + n -= block_size; + BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL + for(difference_type chunk = block_size; chunk; ) { + --chunk; + if(pred(*first)) { + *t_first = *first; + ++t_first; + } + else { + *f_first = *first; + ++f_first; + } + ++first; + } + } - return (partition_copy_leaf) - (r8.first, last, r8.second, t_last, r8.third, f_last, pred, - int()); + //Loop 1 may exhaust an output exactly; do not write past the end. + if(BOOST_UNLIKELY(t_first == t_last)) + goto out_ret; + if(BOOST_UNLIKELY(f_first == f_last)) { + false_output_full = first != last; + goto out_ret; + } + + //Remaining elements + BOOST_CONTAINER_SEGMENTED_UNROLL(4) + for(; n; --n, ++first) { + if(pred(*first)) { + *t_first = *first; + ++t_first; + if(BOOST_UNLIKELY(t_first == t_last)) { + ++first; + goto out_ret; + } + } + else { + *f_first = *first; + ++f_first; + if(BOOST_UNLIKELY(f_first == f_last)) { + ++first; + false_output_full = first != last; + goto out_ret; + } + } + } + out_ret: + return result_t(first, t_first, f_first, false_output_full); } ////////////////////////////////////////////////////////////////////////////// diff --git a/include/boost/container/experimental/segmented_remove_copy.hpp b/include/boost/container/experimental/segmented_remove_copy.hpp index c33c5a9..75ffd9f 100644 --- a/include/boost/container/experimental/segmented_remove_copy.hpp +++ b/include/boost/container/experimental/segmented_remove_copy.hpp @@ -64,46 +64,55 @@ segmented_remove_copy_dst_bounded return segduo(first, dst_first); } -template -BOOST_CONTAINER_FORCEINLINE segtrio -remove_copy_cleanup_blocks - (RASrcIter cur, RADstIter dst_cur, RADstIter dst_last, - const T &value, Diff avail) -{ - const Diff block_size = static_cast(BlockSize); - while(avail >= block_size && - static_cast(dst_last - dst_cur) >= block_size) { - avail -= block_size; - BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL - for(Diff chunk = block_size; chunk; ) { - --chunk; - if(!(*cur == value)) { - transfer_op::apply(*dst_cur, *cur); - ++dst_cur; - } - ++cur; - } - } - return segtrio(cur, dst_cur, avail); -} - template BOOST_CONTAINER_FORCEINLINE typename iterator_enable_if_tag >::type segmented_remove_copy_dst_bounded (RASrcIter first, RASrcIter last, RADstIter dst_first, RADstIter dst_last, const T& value, - const non_segmented_iterator_tag &, const std::random_access_iterator_tag &src_tag) + const non_segmented_iterator_tag &, const std::random_access_iterator_tag &) { + //[alg.remove] same dest-after-write contract as the generic leaf. typedef typename iterator_traits::difference_type difference_type; + typedef segduo result_t; + const difference_type block_size = 16; + difference_type n = last - first; - (void)src_tag; - segtrio r = - (remove_copy_cleanup_blocks<32, Move>) - (first, dst_first, dst_last, value, last - first); + if(BOOST_UNLIKELY(dst_first == dst_last)) + goto out_ret; - return (segmented_remove_copy_dst_bounded) - (r.first, last, r.second, dst_last, value, non_segmented_iterator_tag(), int()); + //Avoid destination check if both input and output ranges are big enough + while(n >= block_size && + static_cast(dst_last - dst_first) >= block_size) { + n -= block_size; + BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL + for(difference_type chunk = block_size; chunk; ) { + --chunk; + if(!(*first == value)) { + transfer_op::apply(*dst_first, *first); + ++dst_first; + } + ++first; + } + } + + //Loop 1 may exhaust the destination exactly; do not write past the end. + if(BOOST_UNLIKELY(dst_first == dst_last)) + goto out_ret; + + //Remaining elements + BOOST_CONTAINER_SEGMENTED_UNROLL(4) + for(; n; --n, ++first) { + if(!(*first == value)) { + transfer_op::apply(*dst_first, *first); + ++dst_first; + if(BOOST_UNLIKELY(dst_first == dst_last)) { + ++first; + goto out_ret; + } + } + } + out_ret: + return result_t(first, dst_first); } template diff --git a/include/boost/container/experimental/segmented_remove_copy_if.hpp b/include/boost/container/experimental/segmented_remove_copy_if.hpp index 1908911..505a9b0 100644 --- a/include/boost/container/experimental/segmented_remove_copy_if.hpp +++ b/include/boost/container/experimental/segmented_remove_copy_if.hpp @@ -77,46 +77,55 @@ segmented_remove_copy_if_dst_bounded return segduo(first, dst_first); } -template -BOOST_CONTAINER_FORCEINLINE segtrio -remove_copy_if_cleanup_blocks - (RASrcIter cur, RADstIter dst_cur, RADstIter dst_last, - Pred pred, Diff avail) -{ - const Diff block_size = static_cast(BlockSize); - while(avail >= block_size && - static_cast(dst_last - dst_cur) >= block_size) { - avail -= block_size; - BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL - for(Diff chunk = block_size; chunk; ) { - --chunk; - if(!pred(*cur)) { - transfer_op::apply(*dst_cur, *cur); - ++dst_cur; - } - ++cur; - } - } - return segtrio(cur, dst_cur, avail); -} - template -typename iterator_enable_if_tag +BOOST_CONTAINER_FORCEINLINE typename iterator_enable_if_tag >::type segmented_remove_copy_if_dst_bounded (RASrcIter first, RASrcIter last, RADstIter dst_first, RADstIter dst_last, Pred pred, - const non_segmented_iterator_tag &, const std::random_access_iterator_tag &src_tag) + const non_segmented_iterator_tag &, const std::random_access_iterator_tag &) { + //[alg.remove] same dest-after-write contract as the generic leaf. typedef typename iterator_traits::difference_type difference_type; + typedef segduo result_t; + const difference_type block_size = 16; + difference_type n = last - first; - (void)src_tag; - segtrio r = - (remove_copy_if_cleanup_blocks<32, Move>) - (first, dst_first, dst_last, pred, last - first); + if(BOOST_UNLIKELY(dst_first == dst_last)) + goto out_ret; - return (segmented_remove_copy_if_dst_bounded) - (r.first, last, r.second, dst_last, pred, non_segmented_iterator_tag(), int()); + //Avoid destination check if both input and output ranges are big enough + while(n >= block_size && + static_cast(dst_last - dst_first) >= block_size) { + n -= block_size; + BOOST_CONTAINER_SEGMENTED_AUTO_UNROLL + for(difference_type chunk = block_size; chunk; ) { + --chunk; + if(!pred(*first)) { + transfer_op::apply(*dst_first, *first); + ++dst_first; + } + ++first; + } + } + + //Loop 1 may exhaust the destination exactly; do not write past the end. + if(BOOST_UNLIKELY(dst_first == dst_last)) + goto out_ret; + + //Remaining elements + BOOST_CONTAINER_SEGMENTED_UNROLL(4) + for(; n; --n, ++first) { + if(!pred(*first)) { + transfer_op::apply(*dst_first, *first); + ++dst_first; + if(BOOST_UNLIKELY(dst_first == dst_last)) { + ++first; + goto out_ret; + } + } + } + out_ret: + return result_t(first, dst_first); } template