fastio
pedis crashed after memtier_benchmark restart
backtrace:
#0 0x000000000074986c in boost::intrusive::list_node_traits<void*>::get_previous (n=@0x7fffffff96c0: 0x8e) at /opt/scylladb/include/boost/intrusive/detail/list_node.hpp:54 #1 0x00000000007dc731 in boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::transfer (p=@0x7fffffff96c0: 0x8e, i=@0x7fffffff96b0: 0x302bddb67028) at /opt/scylladb/include/boost/intrusive/circular_list_algorithms.hpp:284 #2 0x00000000007d6e7b in boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1u>, unsigned long, false, void>::splice (this=0x30000027bb30, p=..., x=..., new_ele=...) at /opt/scylladb/include/boost/intrusive/list.hpp:928 #3 0x00000000007d2542 in log_histogram<logalloc::segment_descriptor, logalloc::segment_descriptor_hist_options>::adjust_up (this=0x30000027b240, v=...) at ./utils/log_histogram.hh:237 #4 0x00000000007cf8de in logalloc::region_impl::free (this=0x30000027b210, obj=0x302bd806ad70, size=24) at utils/logalloc.cc:1354 #5 0x0000000000a3fdb1 in allocation_strategy::destroy<managed<managed_bytes> > (this=0x30000027b210, obj=0x302bd806ad70) at ./utils/allocation_strategy.hh:151 #6 0x0000000000a393f2 in managed_ref<managed_bytes>::~managed_ref (this=0x302bd806ad08, __in_chrg=<optimized out>) at utils/managed_ref.hh:61 #7 0x0000000000a34470 in redis::cache_entry::~cache_entry (this=0x302bd806acf0, __in_chrg=<optimized out>) at cache.hh:179 #8 0x0000000000a3a1b5 in allocation_strategy::destroy<redis::cache_entry> (this=0x30000027b210, obj=0x302bd806acf0) at ./utils/allocation_strategy.hh:150 #9 0x0000000000a34ed4 in auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}::operator()(redis::cache_entry*) const ( __closure=0x7fffffff99e0, obj=0x302bd806acf0) at ./utils/allocation_strategy.hh:202 #10 0x0000000000a4dd05 in boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>::operator()(boost::intrusive::slist_node<void*>* const&) (this=0x7fffffff99e0, p=@0x7fffffff9938: 0x302bd806acf8) at /opt/scylladb/include/boost/intrusive/detail/node_cloner_disposer.hpp:96 #11 0x0000000000a4bb9c in boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void*>*, boost::intrusive::slist_node<void*> >::operator()(boost::intrusive::slist_node<void*>) (this=0x7fffffff99e0, to_clone=0x302bd806acf8) at /opt/scylladb/include/boost/intrusive/hashtable.hpp:570 #12 0x0000000000a47be4 in boost::intrusive::slist_impl<boost::intrusive::trivial_value_traits<boost::intrusive::slist_node_traits<void*>, (boost::intrusive::link_mode_type)0>, unsigned long, 0ul, void>::erase_after_and_dispose<boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void*>*, boost::intrusive::slist_node<void*> > >(boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void*>*, boost::intrusive::slist_node<void*> ><boost::intrusive::trivial_value_traits<boost::intrusive::slist_node_traits<void*>, (boost::intrusive::link_mode_type)0>, true>, boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void*>*, boost::intrusive::slist_node<void*> >) (this=0x302bde053610, prev=..., disposer=...) at /opt/scylladb/include/boost/intrusive/slist.hpp:1053 #13 0x0000000000a41417 in boost::intrusive::bucket_plus_vtraits<boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, boost::intrusive::detail::bucket_traits_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void*> >::type> >::priv_erase_node<boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void*>*, boost::intrusive::slist_node<void*> > >(boost::intrusive::detail::bucket_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void*> >::type>&, boost::intrusive::slist_ite---Type <return> to continue, or q <return> to quit--- rator<boost::intrusive::trivial_value_traits<boost::intrusive::slist_node_traits<void*>, (boost::intrusive::link_mode_type)0>, false>, boost::intrusive::node_cast_adaptor<boost::intrusive::detail::node_disposer<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}, boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, (boost::intrusive::algo_types)1>, boost::intrusive::slist_node<void*>*, boost::intrusive::slist_node<void*> >, boost::move_detail::bool_<false>) (b=..., i=..., node_disposer=...) at /opt/scylladb/include/boost/intrusive/hashtable.hpp:757 #14 0x0000000000a3a5f6 in boost::intrusive::hashtable_impl<boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, void, void, void, boost::intrusive::detail::bucket_traits_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void*> >::type>, unsigned long, 7ul>::erase_and_dispose<auto current_deleter<redis::cache_entry>()::{lambda(redis::cache_entry*)#1}>(boost::intrusive::bucket_plus_vtraits<boost::intrusive::mhtraits<redis::cache_entry, boost::intrusive::unordered_set_member_hook<void, void, void, void>, &redis::cache_entry::_cache_link>, boost::intrusive::detail::bucket_traits_impl<boost::intrusive::detail::get_slist_impl<boost::intrusive::slist_node_traits<void*> >::type> >, boost::move_detail::disable_if_convertible) (this=0x30000023a430, i=..., disposer=...) at /opt/scylladb/include/boost/intrusive/hashtable.hpp:2279 #15 0x0000000000a35490 in redis::cache::insert_if (this=0x30000023a420, entry=0x302bd81e8060, expired=0, nx=false, xx=false) at cache.hh:513 #16 0x00000000009fa9e4 in redis::database::<lambda()>::operator()(void) const (__closure=0x7fffffff9c60) at db.cc:290 #17 0x00000000009faab6 in with_allocator<redis::database::set(redis::redis_key, bytes, long int, uint32_t)::<lambda()> >(allocation_strategy &, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4ce4579, DIE 0x4e85566>) (alloc=..., func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4ce4579, DIE 0x4e85566>) at ./utils/allocation_strategy.hh:271 #18 0x00000000009fabbc in redis::database::set (this=0x30000023a410, rk=..., val=..., expired=0, flag=1) at db.cc:298 #19 0x00000000008d0b3a in std::__invoke_impl<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::* const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(std::__invoke_memfun_deref, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::* const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr<redis::database>&&, redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&&, unsigned char&&) (__f= @0x7fffffffa030: (seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::database * const, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31, true>, long, unsigned int)) 0x9faaea <redis::database::set(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, __t=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab08>, __args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab33>, __args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab42>, __args#2=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab51>, __args#3=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4a8ab60>) at /opt/scylladb/include/c++/7/bits/invoke.h:73 #20 0x00000000008ab9e1 in std::__invoke<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::* const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::* const&)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), seastar::shared_ptr<redis::database>&&, redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&&, unsigned char&&) (__fn= @0x7fffffffa030: (seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::database * const, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31, true>, long, unsigned int)) 0x9faaea <redis::database::set(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, __args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab6312>, ---Type <return> to continue, or q <return> to quit--- __args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab6321>, __args#2=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab6330>, __args#3=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab633f>, __args#4=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab634e>) at /opt/scylladb/include/c++/7/bits/invoke.h:96 #21 0x00000000008998b7 in std::_Mem_fn_base<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), true>::operator()<seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(seastar::shared_ptr<redis::database>&&, redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&&, unsigned char&&) const (this=0x7fffffffa030, __args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada3ed>, __args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada3fc>, __args#2=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada40b>, __args#3=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada41a>, __args#4=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada429>) at /opt/scylladb/include/c++/7/functional:175 #22 0x00000000008871c3 in seastar::apply_helper<std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, std::tuple<seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&, std::integer_sequence<unsigned long, 0ul, 1ul, 2ul, 3ul, 4ul> >::apply(std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>&&, std::tuple<seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd5f9>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd608>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:35 #23 0x000000000088721f in seastar::apply<std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>&&, std::tuple<seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd5bc>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd5ca>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:43 #24 0x000000000088726f in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >::apply<std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>(std::_Mem_fn<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>&&, std::tuple<seastar::shared_ptr<redis::database>, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned char>&&) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd557>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4afd566>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1389 #25 0x000000000087d1ea in seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > seastar::sharded<redis::database>---Type <return> to continue, or q <return> to quit--- ::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}::operator()() (__closure=0x30000010d310) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/sharded.hh:317 #26 0x00000000008998ef in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >::apply<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > seastar::sharded<redis::database>::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}&>(seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > seastar::sharded<redis::database>::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}&) (func=...) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1399 #27 0x000000000088737b in seastar::smp::submit_to<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > seastar::sharded<redis::database>::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}>(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > seastar::sharded<redis::database>::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&)::{lambda()#1}&&) (t=0, func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ab5e5a>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/reactor.hh:1258 #28 0x000000000087d34e in seastar::sharded<redis::database>::invoke_on<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long&, unsigned char&, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >(unsigned int, seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int), redis::redis_key&&, seastar::basic_sstring<char, unsigned int, 31u, true>&&, long&, unsigned char&) (this=0x1218c20 <redis::_databases>, id=0, func= (seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::database::*)(redis::database * const, redis::redis_key, seastar::basic_sstring<char, unsigned int, 31, true>, long, unsigned int)) 0x9faaea <redis::database::set(redis::redis_key, seastar::basic_sstring<char, unsigned int, 31u, true>, long, unsigned int)>, ---Type <return> to continue, or q <return> to quit--- args#0=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada2f4>, args#1=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4710955, DIE 0x4ada304>, args#2=@0x7fffffffa278: 0, args#3=@0x7fffffffa277: 1 '\001') at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/sharded.hh:318 #29 0x0000000000829d02 in redis::redis_service::set (this=0x2aaaaaae1e48, req=...) at redis.cc:135 #30 0x000000000099ff7b in redis::<lambda(redis::request_wrapper&)>::operator()(redis::request_wrapper &) const (__closure=0x30000019eb18, req=...) at server.cc:34 #31 0x00000000009af419 in std::_Function_handler<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >(redis::request_wrapper&), redis::<lambda(redis::request_wrapper&)> >::_M_invoke(const std::_Any_data &, redis::request_wrapper &) (__functor=..., __args#0=...) at /opt/scylladb/include/c++/7/bits/std_function.h:302 #32 0x00000000009d45d5 in std::function<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > (redis::request_wrapper&)>::operator()(redis::request_wrapper&) const (this=0x30000019eb18, __args#0=...) at /opt/scylladb/include/c++/7/bits/std_function.h:706 #33 0x00000000009a4d5d in redis::server::connection::do_handle_one (this=0x3000000c6208, req=...) at server.cc:156 #34 0x00000000009a4de7 in redis::server::connection::<lambda()>::<lambda()>::operator()(void) const (__closure=0x7fffffffa6e0) at server.cc:174 #35 0x00000000009b6722 in seastar::apply_helper<redis::server::connection::handle()::<lambda()>::<lambda()>, std::tuple<>&&, std::integer_sequence<long unsigned int> >::apply(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf097>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf0a5>) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf097>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf0a5>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:35 #36 0x00000000009b676d in seastar::apply<redis::server::connection::handle()::<lambda()>::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf05a>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf068>) (func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf05a>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf068>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:43 #37 0x00000000009b67bd in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >::apply<redis::server::connection::handle()::<lambda()>::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf013>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf022>) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf013>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbf022>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1389 #38 0x00000000009aa256 in seastar::future<>::then<redis::server::connection::handle()::<lambda()>::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b31d>) (this=0x7fffffffa6d0, func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b31d>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:952 #39 0x00000000009a4e41 in redis::server::connection::<lambda()>::operator()(void) const (__closure=0x300000104188) at server.cc:175 #40 0x00000000009b699f in seastar::apply_helper<redis::server::connection::handle()::<lambda()>, std::tuple<>&&, std::integer_sequence<long unsigned int> >::apply(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed0b>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed19>) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed0b>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbed19>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:35 #41 0x00000000009b69ea in seastar::apply<redis::server::connection::handle()::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/relea---Type <return> to continue, or q <return> to quit--- se/pedis, CU 0x4b1e702, DIE 0x4cbecce>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbecdc>) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbecce>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbecdc>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/apply.hh:43 #42 0x00000000009b6a3b in seastar::futurize<seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > > >::apply<redis::server::connection::handle()::<lambda()> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec87>, <unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec96>) ( func=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec87>, args=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4cbec96>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:1389 #43 0x00000000009d1573 in seastar::future<>::<lambda(auto:1&&)>::operator()<seastar::future_state<> >(<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b9fe>) (__closure=0x300000104158, state=<unknown type in /u01/jeff.dk/github/tracymacding/pedis/build/release/pedis, CU 0x4b1e702, DIE 0x4c5b9fe>) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:963 #44 0x00000000009d16d6 in seastar::continuation<seastar::future<T>::then(Func&&) [with Func = redis::server::connection::handle()::<lambda()>; Result = seastar::future<seastar::foreign_ptr<seastar::lw_shared_ptr<seastar::scattered_message<char> > > >; T = {}]::<lambda(auto:1&&)> >::run_and_dispose(void) ( this=0x300000104140) at /u01/jeff.dk/github/tracymacding/pedis/seastar/core/future.hh:414 #45 0x0000000000472a5f in seastar::reactor::run_tasks (this=this@entry=0x300000020000, tq=...) at core/reactor.cc:2699 #46 0x0000000000472d53 in seastar::reactor::run_some_tasks (this=this@entry=0x300000020000) at core/reactor.cc:3122 #47 0x00000000004c6ed0 in run_some_tasks (this=0x300000020000) at /u01/jeff.dk/github/tracymacding/pedis/seastar/util/log.hh:149 #48 seastar::reactor::run (this=0x300000020000) at core/reactor.cc:3269 #49 0x0000000000568d4e in seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) (this=<optimized out>, ac=<optimized out>, av=0x7fffffffb1f8, func=<optimized out>) at core/app-template.cc:185 #50 0x0000000000730603 in main (ac=3, av=0x7fffffffb1f8) at main.cc:81
from debug info, program cores here:
#0 0x0000000000a3ed1d in boost::intrusive::list_node_traits<void*>::get_previous (n=@0x7fffffff96c0: 0x8e) at /opt/scylladb/include/boost/intrusive/detail/list_node.hpp:51 #1 <function called from gdb> #2 0x000000000074986c in boost::intrusive::list_node_traits<void*>::get_previous (n=@0x7fffffff96c0: 0x8e) at /opt/scylladb/include/boost/intrusive/detail/list_node.hpp:54 #3 0x00000000007dc731 in boost::intrusive::circular_list_algorithms<boost::intrusive::list_node_traits<void*> >::transfer (p=@0x7fffffff96c0: 0x8e, i=@0x7fffffff96b0: 0x302bddb67028) at /opt/scylladb/include/boost/intrusive/circular_list_algorithms.hpp:284 #4 0x00000000007d6e7b in boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1u>, unsigned long, false, void>::splice (this=0x30000027bb30, p=..., x=..., new_ele=...) at /opt/scylladb/include/boost/intrusive/list.hpp:928 #5 0x00000000007d2542 in log_histogram<logalloc::segment_descriptor, logalloc::segment_descriptor_hist_options>::adjust_up (this=0x30000027b240, v=...) at ./utils/log_histogram.hh:237
in function adjust_up:
void adjust_up(T& v) { auto b = traits::cached_bucket(v); auto nb = opts.bucket_of(traits::hist_key(v)); if (nb != b) { traits::cache_bucket(v, nb); _buckets[nb].splice(_buckets[nb].begin(), _buckets[b], _buckets[b].iterator_to(v)); _watermark = std::max(ssize_t(nb), _watermark); } }
gdb shows variables value
(gdb) p nb $35 = 143 (gdb) p b $36 = 142 (gdb) p _buckets[nb].begin() $37 = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x30000027bb30}} (gdb) p _buckets[b] $38 = (std::array<boost::intrusive::list<logalloc::segment_descriptor, boost::intrusive::constant_time_size<false>, void, void, void>, 143>::value_type &) @0x30000027bb20: {<boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>, unsigned long, false, void>> = {static constant_time_size = false, static stateful_value_traits = <optimized out>, static has_container_from_iterator = <optimized out>, static safemode_or_autounlink = true, data_ = {<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<logalloc::segment_descriptor, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::safe_link}, root_plus_size_ = {<boost::intrusive::detail::size_holder<false, unsigned long, void>> = {static constant_time_size = <optimized out>}, m_header = {<boost::intrusive::list_node<void*>> = {next_ = 0x302bddb67028, prev_ = 0x302bddb67028}, <No data fields>}}}}, <No data fields>} (gdb) p _buckets[b].iterator_to(v) $39 = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x30000027bb20}}
while in boost splice:
(gdb) f 4 #4 0x00000000007d6e7b in boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1u>, unsigned long, false, void>::splice (this=0x30000027bb30, p=..., x=..., new_ele=...) at /opt/scylladb/include/boost/intrusive/list.hpp:928 928 node_algorithms::transfer(p.pointed_node(), new_ele.pointed_node()); (gdb) p p $40 = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x8e}} (gdb) p x $41 = (boost::intrusive::list_impl<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>, unsigned long, false, void> &) @0x30000027bb20: {static constant_time_size = false, static stateful_value_traits = <optimized out>, static has_container_from_iterator = <optimized out>, static safemode_or_autounlink = true, data_ = {<boost::intrusive::bhtraits<logalloc::segment_descriptor, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)1, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<logalloc::segment_descriptor, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::safe_link}, root_plus_size_ = {<boost::intrusive::detail::size_holder<false, unsigned long, void>> = {static constant_time_size = <optimized out>}, m_header = {<boost::intrusive::list_node<void*>> = {next_ = 0x302bddb67028, prev_ = 0x302bddb67028}, <No data fields>}}}} (gdb) p new_ele $42 = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x302bddb67028}}
it seems the first parameter @p in function splice contains invalid members_(0x8e) which caused the crash.