From e354fb432b947e112b265a9da776b2c41b340383 Mon Sep 17 00:00:00 2001 From: joaquintides Date: Sun, 14 Jul 2024 16:41:21 +0200 Subject: [PATCH 1/5] configured ASAN/TSAN jobs as privileged --- .drone.jsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index b7ff71a8..13fc8453 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -10,8 +10,8 @@ local triggers = }; local ubsan = { UBSAN: '1', UBSAN_OPTIONS: 'print_stacktrace=1' }; -local asan = { ASAN: '1' }; -local tsan = { TSAN: '1' }; +local asan = { ASAN: '1', ASAN_OPTIONS: 'privileged=True' }; +local tsan = { TSAN: '1' TSAN_OPTIONS: 'privileged=True' }; local linux_pipeline(name, image, environment, packages = "", sources = [], arch = "amd64") = { From ac1a2b4d23bdfabfb981fd79ae8152c7fa5631ab Mon Sep 17 00:00:00 2001 From: joaquintides Date: Sun, 14 Jul 2024 16:43:04 +0200 Subject: [PATCH 2/5] fixed syntax error --- .drone.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 13fc8453..6a6d4b26 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -11,7 +11,7 @@ local triggers = local ubsan = { UBSAN: '1', UBSAN_OPTIONS: 'print_stacktrace=1' }; local asan = { ASAN: '1', ASAN_OPTIONS: 'privileged=True' }; -local tsan = { TSAN: '1' TSAN_OPTIONS: 'privileged=True' }; +local tsan = { TSAN: '1', TSAN_OPTIONS: 'privileged=True' }; local linux_pipeline(name, image, environment, packages = "", sources = [], arch = "amd64") = { From 3ad8ac4c34a30be3c1b5a0ef877512067003fafa Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 15 Jul 2024 13:38:32 +0300 Subject: [PATCH 3/5] fix asan and tsan jobs on Drone (#264) --- .drone.jsonnet | 5 +++-- .drone/drone.sh | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 6a6d4b26..749d513d 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -10,8 +10,8 @@ local triggers = }; local ubsan = { UBSAN: '1', UBSAN_OPTIONS: 'print_stacktrace=1' }; -local asan = { ASAN: '1', ASAN_OPTIONS: 'privileged=True' }; -local tsan = { TSAN: '1', TSAN_OPTIONS: 'privileged=True' }; +local asan = { ASAN: '1' }; +local tsan = { TSAN: '1' }; local linux_pipeline(name, image, environment, packages = "", sources = [], arch = "amd64") = { @@ -30,6 +30,7 @@ local linux_pipeline(name, image, environment, packages = "", sources = [], arch name: "everything", image: image, environment: environment, + privileged: true, commands: [ 'set -e', diff --git a/.drone/drone.sh b/.drone/drone.sh index 60f06760..ed7837bb 100755 --- a/.drone/drone.sh +++ b/.drone/drone.sh @@ -14,6 +14,11 @@ DRONE_BUILD_DIR=$(pwd) BOOST_BRANCH=develop if [ "$DRONE_BRANCH" = "master" ]; then BOOST_BRANCH=master; fi +if [[ $(uname) == "Linux" && ( "$TSAN" == 1 || "$ASAN" == 1 ) ]]; then + echo 0 | sudo tee /proc/sys/kernel/randomize_va_space + sudo sysctl vm.mmap_rnd_bits=28 +fi + cd .. git clone -b $BOOST_BRANCH --depth 1 https://github.com/boostorg/boost.git boost-root cd boost-root From 7ddd562532326a05252f08e019d60e62e2cdd347 Mon Sep 17 00:00:00 2001 From: Braden Ganetsky Date: Mon, 15 Jul 2024 12:42:28 -0500 Subject: [PATCH 4/5] Implement natvis for fancy pointers (#262) * Add file for manual natvis testing * Equip natvis file to allow fancy pointers * Update docs * [skip ci] add links to natvis docs --- doc/unordered/changes.adoc | 4 + doc/unordered/debuggability.adoc | 14 +- extra/boost_unordered.natvis | 218 +++++++++++++++----- include/boost/unordered/detail/foa/core.hpp | 5 + test/Jamfile.v2 | 2 + test/natvis_tests.cpp | 187 +++++++++++++++++ 6 files changed, 382 insertions(+), 48 deletions(-) create mode 100644 test/natvis_tests.cpp diff --git a/doc/unordered/changes.adoc b/doc/unordered/changes.adoc index ffdaf242..550a7068 100644 --- a/doc/unordered/changes.adoc +++ b/doc/unordered/changes.adoc @@ -6,6 +6,10 @@ :github-pr-url: https://github.com/boostorg/unordered/pull :cpp: C++ +== Release 1.87.0 + +* In Visual Studio Natvis, supported any container with an allocator that uses fancy pointers. This applies to any fancy pointer type, as long as the proper Natvis customization point "Intrinsic" functions are written for the fancy pointer type. + == Release 1.86.0 * Added container `pmr` aliases when header `` is available. The alias `boost::unordered::pmr::[container]` refers to `boost::unordered::[container]` with a `std::pmr::polymorphic_allocator` allocator type. diff --git a/doc/unordered/debuggability.adoc b/doc/unordered/debuggability.adoc index cd40e498..319ab54e 100644 --- a/doc/unordered/debuggability.adoc +++ b/doc/unordered/debuggability.adoc @@ -5,7 +5,13 @@ == Visual Studio Natvis -All containers and iterators have custom visualizations in the Natvis framework, as long as their allocator uses regular raw pointers. Any container or iterator with an allocator using fancy pointers does not have a custom visualization right now. +All containers and iterators have custom visualizations in the Natvis framework. + +=== Using in your project + +To visualize Boost.Unordered containers in the Natvis framework in your project, simply add the file link:https://github.com/boostorg/unordered/blob/develop/extra/boost_unordered.natvis[/extra/boost_unordered.natvis] to your Visual Studio project as an "Existing Item". + +=== Visualization structure The visualizations mirror those for the standard unordered containers. A container has a maximum of 100 elements displayed at once. Each set element has its item name listed as `[i]`, where `i` is the index in the display, starting at `0`. Each map element has its item name listed as `[\{key-display}]` by default. For example, if the first element is the pair `("abc", 1)`, the item name will be `["abc"]`. This behaviour can be overridden by using the view "ShowElementsByIndex", which switches the map display behaviour to name the elements by index. This same view name is used in the standard unordered containers. @@ -13,4 +19,8 @@ By default, the closed-addressing containers will show the `[hash_function]` and By default, the open-addressing containers will show the `[hash_function]`, `[key_eq]`, `[allocator]`, and the elements. Using the view "simple" shows only the elements, with no other items present. Both the SIMD and the non-SIMD implementations are viewable through the Natvis framework. -Iterators are displayed similarly to their standard counterparts. An iterator is displayed as though it were the element that it points to. An end iterator is simply displayed as `\{ end iterator }`. +Iterators are displayed similarly to their standard counterparts. An iterator is displayed as though it were the element that it points to. An end iterator is simply displayed as `{ end iterator }`. + +=== Fancy pointers + +The container visualizations also work if you are using fancy pointers in your allocator, such as `boost::interprocess::offset_ptr`. While this is rare, Boost.Unordered has natvis customization points to support any type of fancy pointer. `boost::interprocess::offset_ptr` has support already defined in the Boost.Interprocess library, and you can add support to your own type by following the instructions contained in a comment near the end of the file link:https://github.com/boostorg/unordered/blob/develop/extra/boost_unordered.natvis[/extra/boost_unordered.natvis]. diff --git a/extra/boost_unordered.natvis b/extra/boost_unordered.natvis index 58ba0427..75df78c7 100644 --- a/extra/boost_unordered.natvis +++ b/extra/boost_unordered.natvis @@ -30,18 +30,50 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + + + + + + + + + + + + + + + + + + + + + + + - - + + - current_bucket = &buckets[bucket_index] - node = current_bucket->next + current_bucket = next(to_address(&buckets), bucket_index) + node = to_address(&current_bucket->next) node->buf.t_ - node = node->next + node = to_address(&node->next) ++bucket_index @@ -54,14 +86,14 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - - + + - current_bucket = &buckets[bucket_index] - node = current_bucket->next + current_bucket = next(to_address(&buckets), bucket_index) + node = to_address(&current_bucket->next) node->buf.t_ - node = node->next + node = to_address(&node->next) ++bucket_index @@ -99,29 +131,46 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - - {p->buf.t_} + + + + + + + + + + + + + + + + + + + + {to_address(&p)->buf.t_} {{ end iterator }} - p->buf.t_ + to_address(&p)->buf.t_ - {*p} + + + + + + + + + {*get()} - *p - - - - - - ({p->first}, {p->second}) - - p->first - p->second + *get() @@ -227,10 +276,39 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -254,8 +332,8 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) cstats - - + + @@ -264,18 +342,18 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - + *p_ first_time = false n0 = reinterpret_cast<uintptr_t>(pc_) % sizeof(group_type) - pc_ -= (ptrdiff_t)n0 + pc_ = next(pc_, -(ptrdiff_t)n0) mask = (reinterpret_cast<group_type*>(pc_)->match_occupied() >> (n0+1)) << (n0+1) - pc_ += sizeof(group_type) - p_ += group_type::N + pc_ = next(pc_, sizeof(group_type)) + p_ = next(p_, group_type::N) mask = reinterpret_cast<group_type*>(pc_)->match_occupied() @@ -284,9 +362,8 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) p_ = nullptr - pc_ += (ptrdiff_t)n - p_ -= (ptrdiff_t)n0 - p_ += (ptrdiff_t)n + pc_ = next(pc_, (ptrdiff_t)n) + p_ = next(p_, (ptrdiff_t)n - (ptrdiff_t)n0) @@ -299,8 +376,8 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) cstats - - + + @@ -309,18 +386,18 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - + *p_ first_time = false n0 = reinterpret_cast<uintptr_t>(pc_) % sizeof(group_type) - pc_ -= (ptrdiff_t)n0 + pc_ = next(pc_, -(ptrdiff_t)n0) mask = (reinterpret_cast<group_type*>(pc_)->match_occupied() >> (n0+1)) << (n0+1) - pc_ += sizeof(group_type) - p_ += group_type::N + pc_ = next(pc_, sizeof(group_type)) + p_ = next(p_, group_type::N) mask = reinterpret_cast<group_type*>(pc_)->match_occupied() @@ -329,9 +406,8 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) p_ = nullptr - pc_ += (ptrdiff_t)n - p_ -= (ptrdiff_t)n0 - p_ += (ptrdiff_t)n + pc_ = next(pc_, (ptrdiff_t)n) + p_ = next(p_, (ptrdiff_t)n - (ptrdiff_t)n0) @@ -369,12 +445,62 @@ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - - {*p_} + + + + + + + + + + + + + + + {*to_address(&p_)} {{ end iterator }} - *p_ + *to_address(&p_) + + + + diff --git a/include/boost/unordered/detail/foa/core.hpp b/include/boost/unordered/detail/foa/core.hpp index e89a21cf..3da26fef 100644 --- a/include/boost/unordered/detail/foa/core.hpp +++ b/include/boost/unordered/detail/foa/core.hpp @@ -1017,6 +1017,11 @@ struct table_arrays rebind; using group_type_pointer_traits=boost::pointer_traits; + // For natvis purposes + using char_pointer= + typename boost::pointer_traits::template + rebind; + table_arrays( std::size_t gsi,std::size_t gsm, group_type_pointer pg,value_type_pointer pe): diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 2b46f4f7..418820a4 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -42,6 +42,8 @@ path-constant BOOST_UNORDERED_TEST_DIR : . ; run quick.cpp ; +compile natvis_tests.cpp ; + compile unordered/self_include_tests_obj.cpp : BOOST_UNORDERED_HEADER="boost/unordered_map.hpp" : tl_unordered_map_hpp ; diff --git a/test/natvis_tests.cpp b/test/natvis_tests.cpp new file mode 100644 index 00000000..4a420b80 --- /dev/null +++ b/test/natvis_tests.cpp @@ -0,0 +1,187 @@ +// Copyright 2024 Braden Ganetsky +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +// This test applies to MSVC only. This is a file for manual testing. +// Run this test and break manually at the variable called `break_here`. +// Inspect the variables using the Visual Studio debugger to test correctness. + +#if !defined(BOOST_MSVC) + +#include +BOOST_PRAGMA_MESSAGE("These tests are for Visual Studio only.") +int main() {} + +#else + +#if 0 // Change to `#if 1` to test turning off SIMD optimizations +#define BOOST_UNORDERED_DISABLE_SSE2 +#define BOOST_UNORDERED_DISABLE_NEON +#endif + +#define BOOST_UNORDERED_ENABLE_STATS + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Prevent any "unused" errors +template void use(Args&&...) {} + +using map_value_type = std::pair; +using set_value_type = std::string; + +template void natvis_test(Tester& tester) +{ + // clang-format off + auto fca_map = tester.template construct_map(); + auto fca_multimap = tester.template construct_map(); + auto fca_set = tester.template construct_set(); + auto fca_multiset = tester.template construct_set(); + + auto foa_flat_map = tester.template construct_map(); + auto foa_flat_set = tester.template construct_set(); + auto foa_node_map = tester.template construct_map(); + auto foa_node_set = tester.template construct_set(); + + auto cfoa_flat_map = tester.template construct_map(); + auto cfoa_flat_set = tester.template construct_set(); + // clang-format on + + for (int i = 0; i < 5; ++i) { + const auto str = std::to_string(i * 2); + const auto num = i * 11; + + fca_map->emplace(str, num); + fca_multimap->emplace(str, num); + fca_multimap->emplace(str, num + 1); + foa_flat_map->emplace(str, num); + foa_node_map->emplace(str, num); + cfoa_flat_map->emplace(str, num); + + fca_set->emplace(str); + fca_multiset->emplace(str); + fca_multiset->emplace(str); + foa_flat_set->emplace(str); + foa_node_set->emplace(str); + cfoa_flat_set->emplace(str); + } + + auto fca_map_begin = fca_map->begin(); + auto fca_map_end = fca_map->end(); + auto fca_multimap_begin = fca_multimap->begin(); + auto fca_multimap_end = fca_multimap->end(); + auto fca_set_begin = fca_set->begin(); + auto fca_set_end = fca_set->end(); + auto fca_multiset_begin = fca_multiset->begin(); + auto fca_multiset_end = fca_multiset->end(); + + auto foa_flat_map_begin = foa_flat_map->begin(); + auto foa_flat_map_end = foa_flat_map->end(); + auto foa_flat_set_begin = foa_flat_set->begin(); + auto foa_flat_set_end = foa_flat_set->end(); + auto foa_node_map_begin = foa_node_map->begin(); + auto foa_node_map_end = foa_node_map->end(); + auto foa_node_set_begin = foa_node_set->begin(); + auto foa_node_set_end = foa_node_set->end(); + + use(cfoa_flat_map, cfoa_flat_set); + use(fca_map_begin, fca_map_end, fca_multimap_begin, fca_multimap_end, + fca_set_begin, fca_set_end, fca_multiset_begin, fca_multiset_end); + use(foa_flat_map_begin, foa_flat_map_end, foa_flat_set_begin, + foa_flat_set_end, foa_node_map_begin, foa_node_map_end, foa_node_set_begin, + foa_node_set_end); + + int break_here = 0; + use(break_here); +} + +class offset_ptr_tester_ +{ + static constexpr std::size_t SEGMENT_SIZE = 64 * 1024; + std::string segment_name = to_string(boost::uuids::random_generator()()); + boost::interprocess::managed_shared_memory segment{ + boost::interprocess::create_only, segment_name.c_str(), SEGMENT_SIZE}; + + using map_allocator = boost::interprocess::allocator; + using set_allocator = boost::interprocess::allocator; + + template