teracli would core dump when exit

[Liufeiran123]

#0  0x00007f2a47a45cc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1  0x00007f2a47a490d8 in __GI_abort () at abort.c:89
#2  0x00007f2a48350535 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3  0x00007f2a4834e6d6 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#4  0x00007f2a4834e703 in std::terminate() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#5  0x00007f2a4834f1bf in __cxa_pure_virtual () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#6  0x00000000008fe7e5 in boost::system::error_code::message (this=0x7f2a44206c70)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/system/error_code.hpp:357
#7  0x0000000000932146 in sofa::pbrpc::RpcMessageStream<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcControllerImpl> >::on_read_some (
    this=0x2291400, error=..., bytes_transferred=0) at src/sofa/pbrpc/rpc_message_stream.h:237
#8  0x00000000009304f1 in boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>::call<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream>, boost::system::error_code const, unsigned long> (this=0x7f2a44206c50, u=..., b1=..., 
    b2=@0x7f2a442069a0: 0) at /home/lfr/tera/thirdparty/boost_1_57_0/boost/bind/mem_fn_template.hpp:271
#9  0x00000000009302ab in boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>::operator()<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> > (this=0x7f2a44206c50, u=..., a1=..., a2=0)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/bind/mem_fn_template.hpp:286
#10 0x000000000092fdb2 in boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> >::operator()<boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list2<boost::system::error_code const&, unsigned long const&> > (this=0x7f2a44206c60, f=..., a=...)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/bind/bind.hpp:392
#11 0x000000000092f8f4 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> > >::operator()<boost::system::error_code, unsigned long> (this=0x7f2a44206c50, a1=..., a2=@0x7f2a44206c80: 0)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/bind/bind_template.hpp:102
#12 0x000000000092efcd in boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> > >, boost::system::error_code, unsigned long>::operator() (this=0x7f2a44206c50)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/bind_handler.hpp:127
#13 0x000000000092d690 in boost::asio::asio_handler_invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> > >, boost::system::error_code, unsigned long> > (function=...)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/handler_invoke_hook.hpp:69
#14 0x000000000092acda in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> > > > (function=..., context=...)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/handler_invoke_helpers.hpp:37
---Type <return> to continue, or q <return> to quit---
#15 0x00000000009281ae in boost::asio::detail::reactive_socket_recv_op<boost::asio::mutable_buffers_1, boost::_bi::bind_t<void, boost::_mfi::mf2<void, sofa::pbrpc::RpcByteStream, boost::system::error_code const&, unsigned long>, boost::_bi::list3<boost::_bi::value<sofa::pbrpc::shared_ptr<sofa::pbrpc::RpcByteStream> >, boost::arg<1>, boost::arg<2> > > >::do_complete (owner=0x220e1a0, base=0x2232980)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/reactive_socket_recv_op.hpp:110
#16 0x0000000000905270 in boost::asio::detail::task_io_service_operation::complete (this=0x2232980, owner=..., ec=..., bytes_transferred=0)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/task_io_service_operation.hpp:38
#17 0x00000000009071cd in boost::asio::detail::epoll_reactor::descriptor_state::do_complete (owner=0x220e1a0, base=0x21f2780, ec=..., 
    bytes_transferred=21) at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/impl/epoll_reactor.ipp:651
#18 0x0000000000905270 in boost::asio::detail::task_io_service_operation::complete (this=0x21f2780, owner=..., ec=..., bytes_transferred=21)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/task_io_service_operation.hpp:38
#19 0x0000000000907cca in boost::asio::detail::task_io_service::do_run_one (this=0x220e1a0, lock=..., this_thread=..., ec=...)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/impl/task_io_service.ipp:372
#20 0x000000000090782b in boost::asio::detail::task_io_service::run (this=0x220e1a0, ec=...)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/detail/impl/task_io_service.ipp:149
#21 0x0000000000907fd7 in boost::asio::io_service::run (this=0x22060c0)
    at /home/lfr/tera/thirdparty/boost_1_57_0/boost/asio/impl/io_service.ipp:59
#22 0x000000000090e6d6 in sofa::pbrpc::ThreadGroupImpl::thread_run (param=0x221e0e0) at src/sofa/pbrpc/thread_group_impl.h:263
#23 0x00007f2a48815182 in start_thread (arg=0x7f2a44207700) at pthread_create.c:312
#24 0x00007f2a47b0947d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

[bluebore]

辛苦 @caijieming-baidu 跟进这个问题， 辛苦 @00k 协助

[yvxiang]

看起来和我上次遇到的差不多，sofa版本是多少？

[caijieming-ng]

@yvxiang ,你执行什么操作错core的

[caijieming-ng]

@lfrdreamman，非常感谢对tera的支持；请问你的运行环境是什么（Ubuntu，centos，或...）？跑onebox时core掉吗？执行什么样的teracli操作导致出core的？我这边希望根据你的反馈追一下问题。

[yvxiang]

@caijieming-baidu 所有操作都会core. 是全局变量的析构问题，和sofa有关，后来 @cyshi 同学解决了，上面那个链接里有详细的原因以复现方法

[imotai]

遇到同样的问题, haha

[Liufeiran123]

在ubuntu环境下，跑onebox时core掉，所有teracli操作都会core，应该是全局变量的析构顺序问题。

[Liufeiran123]

看之前的issue，这个问题已经解决，应该是我sofa版本的问题。