dragonfly
dragonfly copied to clipboard
Published
20 hours ago •
dragonflydb
dragonflydb
ordering bug inside our synchronization primitives
F20230620 23:30:37.838975 12682 init.cc:21] [/usr/include/boost/intrusive/slist.hpp:460]: assert(!safemode_or_autounlink || node_algorithms::inited(n)) failed!
*** Check failure stack trace: ***
@ 0xaaaaaddbedbc google::LogMessage::Fail()
@ 0xaaaaaddbecc8 google::LogMessage::SendToLog()
@ 0xaaaaaddbe550 google::LogMessage::Flush()
@ 0xaaaaaddc1d18 google::LogMessageFatal::~LogMessageFatal()
@ 0xaaaaac950f5c __assert_fail
@ 0xaaaaac7e08c0 boost::intrusive::slist_impl<>::push_back()
@ 0xaaaaac7ca134 util::fb2::EventCount::wait()
@ 0xaaaaad706d70 _ZN4util3fb210EventCount5awaitIZN4dfly11Transaction21WaitForShardCallbacksEvEUlvE_EEbT_
@ 0xaaaaad705260 dfly::Transaction::WaitForShardCallbacks()
@ 0xaaaaad6e1dfc dfly::Transaction::UnwatchBlocking()
@ 0xaaaaad6e7984 dfly::Transaction::WaitOnWatch()
@ 0xaaaaacf0ba70 dfly::container_utils::RunCbOnFirstNonEmptyBlocking()
#4 0x0000aaaaaddc7aec in google::DumpStackTraceAndExit () at /home/dev/projects/dragonfly/build-dbg/_deps/glog-src/src/utilities.cc:178
#5 0x0000aaaaaddbedbc in google::LogMessage::Fail () at /home/dev/projects/dragonfly/build-dbg/_deps/glog-src/src/logging.cc:1949
#6 0x0000aaaaaddbecc8 in google::LogMessage::SendToLog (this=0xffffe13512f8) at /home/dev/projects/dragonfly/build-dbg/_deps/glog-src/src/logging.cc:1923
#7 0x0000aaaaaddbe550 in google::LogMessage::Flush (this=0xffffe13512f8) at /home/dev/projects/dragonfly/build-dbg/_deps/glog-src/src/logging.cc:1780
#8 0x0000aaaaaddc1d18 in google::LogMessageFatal::~LogMessageFatal (this=0xffffe13512f8, __in_chrg=<optimized out>) at /home/dev/projects/dragonfly/build-dbg/_deps/glog-src/src/logging.cc:2560
#9 0x0000aaaaac950f5c in __assert_fail (assertion=0xaaaaadf688f0 "!safemode_or_autounlink || node_algorithms::inited(n)", file=0xaaaaadf66ad0 "/usr/include/boost/intrusive/slist.hpp", line=460,
function=0xaaaaadf68720 "void boost::intrusive::slist_impl<ValueTraits, SizeType, BoolFlags, HeaderHolder>::push_back(reference) [with ValueTraits = boost::intrusive::mhtraits<util::fb2::detail::FiberInterface, boost::intrusi"...)
at /home/dev/projects/dragonfly/helio/base/init.cc:21
#10 0x0000aaaaac7e08c0 in boost::intrusive::slist_impl<boost::intrusive::mhtraits<util::fb2::detail::FiberInterface, boost::intrusive::slist_member_hook<boost::intrusive::link_mode<(boost::intrusive::link_mode_type)1> >, &util::fb2::detai
l::FiberInterface::wait_hook>, unsigned long, 4ul, void>::push_back (this=0xffffe06dec00, value=...) at /usr/include/boost/intrusive/slist.hpp:460
#11 0x0000aaaaac7ca134 in util::fb2::EventCount::wait (this=0xffffe06debf0, epoch=2) at /home/dev/projects/dragonfly/helio/util/fibers/synchronization.h:551
#12 0x0000aaaaad706d70 in util::fb2::EventCount::await<dfly::Transaction::WaitForShardCallbacks()::{lambda()#1}>(dfly::Transaction::WaitForShardCallbacks()::{lambda()#1}) (this=0xffffe06debf0, condition=...)
at /home/dev/projects/dragonfly/helio/util/fibers/synchronization.h:600
#13 0x0000aaaaad705260 in dfly::Transaction::WaitForShardCallbacks (this=0xffffe06dea00) at /home/dev/projects/dragonfly/src/server/transaction.h:467
#14 0x0000aaaaad6e1dfc in dfly::Transaction::UnwatchBlocking(bool, std::function<absl::lts_20230125::Span<std::basic_string_view<char, std::char_traits<char> > const> (dfly::Transaction*, dfly::EngineShard*)>) (this=0xffffe06dea00,
should_expire=false, wcb=...) at /home/dev/projects/dragonfly/src/server/transaction.cc:932
#15 0x0000aaaaad6e7984 in dfly::Transaction::WaitOnWatch(std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > const&, std::function<absl::lts_20230125::Span<std::basic_string_
view<char, std::char_traits<char> > const> (dfly::Transaction*, dfly::EngineShard*)>) (this=0xffffe06dea00, tp=std::chrono::_V2::steady_clock time_point = {139077694898401ns}, wkeys_provider=...)
at /home/dev/projects/dragonfly/src/server/transaction.cc:1147
#16 0x0000aaaaacf0ba70 in dfly::container_utils::RunCbOnFirstNonEmptyBlocking(std::function<void (dfly::Transaction*, dfly::EngineShard*, std::basic_string_view<char, std::char_traits<char> >)>&&, std::__cxx11::basic_string<char, std::cha
r_traits<char>, std::allocator<char> >*, dfly::Transaction*, int, unsigned int) (func=..., out_key=0xffffe1351a28, trans=0xffffe06dea00, req_obj_type=1, limit_ms=2000) at /home/dev/projects/dragonfly/src/server/container_utils.cc:349
#17 0x0000aaaaad4a3be0 in dfly::ListFamily::BPopGeneric (dir=dfly::ListDir::LEFT, args=..., cntx=0xffffe128ccd0) at /home/dev/projects/dragonfly/src/server/list_family.cc:1209
For now - submitted https://github.com/romange/helio/commit/5aa7d7ccd1e7c83af6026cb1e52fe7117a5e807e to check if we somehow ended up in inconsistent state