redpanda icon indicating copy to clipboard operation
redpanda copied to clipboard

Published 20 hours ago • verified publisher icon redpanda-data

Failure in `NodesDecommissioningTest.test_recommissioning_one_of_decommissioned_nodes`

Open NyaliaLui opened this issue 3 years ago • 10 comments

Seen in https://buildkite.com/redpanda/redpanda/builds/16483#0183c7b1-90d7-435c-9c27-da7fed969939/6-8059 Specifically the PR #6639

test_id:    rptest.tests.nodes_decommissioning_test.NodesDecommissioningTest.test_recommissioning_one_of_decommissioned_nodes
status:     FAIL
run time:   2 minutes 32.580 seconds
 
    TimeoutError('')
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/ducktape/tests/runner_client.py", line 135, in run
    data = self.run_test()
  File "/usr/local/lib/python3.10/dist-packages/ducktape/tests/runner_client.py", line 227, in run_test
    return self.test_context.function(self.test)
  File "/root/tests/rptest/services/cluster.py", line 35, in wrapped
    r = f(self, *args, **kwargs)
  File "/root/tests/rptest/tests/nodes_decommissioning_test.py", line 364, in test_recommissioning_one_of_decommissioned_nodes
    wait_until(node_removed, 60, 2)
  File "/usr/local/lib/python3.10/dist-packages/ducktape/utils/util.py", line 58, in wait_until
    raise TimeoutError(err_msg() if callable(err_msg) else err_msg) from last_exception
ducktape.errors.TimeoutError

NyaliaLui avatar Oct 11 '22 19:10 NyaliaLui

There is a backtrace on docker-rp-11 for that test

detail::operator<<(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const&) at ??:?
raft::operator<<(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, raft::vnode const&) at ??:?
std::__1::basic_ostream<char, std::__1::char_traits<char> >& std::operator<< <raft::vnode>(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&) at ??:?
void fmt::v8::detail::format_value<char, std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > >(fmt::v8::detail::buffer<char>&, std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&, fmt::v8::detail::locale_ref) at ??:?
fmt::v8::appender fmt::v8::detail::fallback_formatter<std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> >, char, void>::format<fmt::v8::appender>(std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at ??:?
void fmt::v8::detail::value<fmt::v8::basic_format_context<fmt::v8::appender, char> >::format_custom_arg<std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> >, fmt::v8::detail::fallback_formatter<std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> >, char, void> >(void*, fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at ??:?
fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle::format(fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) const at application.cc:?
fmt::v8::detail::default_arg_formatter<char>::operator()(fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle) at application.cc:?
char const* fmt::v8::detail::parse_replacement_field<char, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&>(char const*, char const*, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&) at application.cc:?
void fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref) at application.cc:?
void fmt::v8::vprint<char>(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >) at application.cc:?
void fmt::v8::print<char [29], std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&, std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&, char>(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, char const (&) [29], std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&, std::__1::vector<raft::vnode, std::__1::allocator<raft::vnode> > const&) at ??:?
raft::operator<<(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, raft::group_nodes const&) at ??:?
void fmt::v8::detail::format_value<char, raft::group_nodes>(fmt::v8::detail::buffer<char>&, raft::group_nodes const&, fmt::v8::detail::locale_ref) at ??:?
fmt::v8::appender fmt::v8::detail::fallback_formatter<raft::group_nodes, char, void>::format<fmt::v8::appender>(raft::group_nodes const&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at ??:?
void fmt::v8::detail::value<fmt::v8::basic_format_context<fmt::v8::appender, char> >::format_custom_arg<raft::group_nodes, fmt::v8::detail::fallback_formatter<raft::group_nodes, char, void> >(void*, fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at ??:?
fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle::format(fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) const at application.cc:?
fmt::v8::detail::default_arg_formatter<char>::operator()(fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle) at application.cc:?
char const* fmt::v8::detail::parse_replacement_field<char, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&>(char const*, char const*, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&) at application.cc:?
void fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref) at application.cc:?
void fmt::v8::vprint<char>(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >) at application.cc:?
void fmt::v8::print<char [76], raft::group_nodes const&, std::__1::optional<raft::group_nodes> const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > const&, std::__1::optional<raft::configuration_update> const&, detail::base_named_type<signed char, raft::raft_group_configuration_version, std::__1::integral_constant<bool, true> > const&, std::__1::vector<model::broker, std::__1::allocator<model::broker> > const&, char>(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, char const (&) [76], raft::group_nodes const&, std::__1::optional<raft::group_nodes> const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > const&, std::__1::optional<raft::configuration_update> const&, detail::base_named_type<signed char, raft::raft_group_configuration_version, std::__1::integral_constant<bool, true> > const&, std::__1::vector<model::broker, std::__1::allocator<model::broker> > const&) at ??:?
raft::operator<<(std::__1::basic_ostream<char, std::__1::char_traits<char> >&, raft::group_configuration const&) at ??:?
void fmt::v8::detail::format_value<char, raft::group_configuration>(fmt::v8::detail::buffer<char>&, raft::group_configuration const&, fmt::v8::detail::locale_ref) at ??:?
fmt::v8::appender fmt::v8::detail::fallback_formatter<raft::group_configuration, char, void>::format<fmt::v8::appender>(raft::group_configuration const&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at ??:?
void fmt::v8::detail::value<fmt::v8::basic_format_context<fmt::v8::appender, char> >::format_custom_arg<raft::group_configuration, fmt::v8::detail::fallback_formatter<raft::group_configuration, char, void> >(void*, fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at ??:?
fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle::format(fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) const at application.cc:?
fmt::v8::detail::default_arg_formatter<char>::operator()(fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle) at application.cc:?
char const* fmt::v8::detail::parse_replacement_field<char, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&>(char const*, char const*, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&) at application.cc:?
void fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref) at application.cc:?
seastar::internal::log_buf::inserter_iterator fmt::v8::vformat_to<seastar::internal::log_buf::inserter_iterator, 0>(seastar::internal::log_buf::inserter_iterator, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<fmt::v8::appender, char> >) at application.cc:?
seastar::logger::log<char const*, int, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&>(seastar::log_level, seastar::logger::format_info, char const*&&, int&&, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&)::{lambda(seastar::internal::log_buf::inserter_iterator)#1}::operator()(seastar::internal::log_buf::inserter_iterator) const at ??:?
seastar::logger::lambda_log_writer<seastar::logger::log<char const*, int, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&>(seastar::log_level, seastar::logger::format_info, char const*&&, int&&, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&)::{lambda(seastar::internal::log_buf::inserter_iterator)#1}>::operator()(seastar::internal::log_buf::inserter_iterator) at ??:?
seastar::logger::do_log(seastar::log_level, seastar::logger::log_writer&)::$_0::operator()(seastar::internal::log_buf::inserter_iterator) const at log.cc:?
seastar::logger::do_log(seastar::log_level, seastar::logger::log_writer&) at log.cc:?
void seastar::logger::log<char const*, int, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&>(seastar::log_level, seastar::logger::format_info, char const*&&, int&&, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&) at ??:?
void seastar::logger::trace<char const*, int, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&>(seastar::logger::format_info, char const*&&, int&&, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&) at ??:?
cluster::check_configuration_update(detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, seastar::lw_shared_ptr<cluster::partition> const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
seastar::future<std::__1::error_code> cluster::controller_backend::apply_configuration_change_on_leader<cluster::controller_backend::update_partition_replica_set(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >)::$_27>(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, cluster::controller_backend::update_partition_replica_set(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >)::$_27&&) at controller_backend.cc:?
cluster::controller_backend::update_partition_replica_set(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
cluster::controller_backend::execute_reconfiguration(cluster::topic_table_delta::op_type, model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
cluster::controller_backend::process_partition_reconfiguration(cluster::topic_table_delta::op_type, model::ntp, cluster::partition_assignment const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
cluster::controller_backend::execute_partition_op(cluster::topic_table_delta const&) at ??:?
cluster::controller_backend::reconcile_ntp(std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> >&) at ??:?
cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}::operator()(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&) const at controller_backend.cc:?
seastar::future<void> seastar::futurize<seastar::future<void> >::invoke<cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&>(cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}&&, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&) at controller_backend.cc:?
auto seastar::futurize_invoke<cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&>(cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}&&, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&) at controller_backend.cc:?
seastar::future<void> seastar::parallel_for_each<absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}>(absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}&&) at controller_backend.cc:?
cluster::controller_backend::reconcile_topics()::$_21::operator()() const at controller_backend.cc:?
seastar::future<void> seastar::futurize<seastar::future<void> >::invoke<cluster::controller_backend::reconcile_topics()::$_21>(cluster::controller_backend::reconcile_topics()::$_21&&) at controller_backend.cc:?
auto seastar::futurize_invoke<cluster::controller_backend::reconcile_topics()::$_21>(cluster::controller_backend::reconcile_topics()::$_21&&) at controller_backend.cc:?
auto seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}::operator()<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>) at controller_backend.cc:?
seastar::future<void> seastar::futurize<seastar::future<void> >::invoke<seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at controller_backend.cc:?
auto seastar::futurize_invoke<seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at controller_backend.cc:?
_ZZN7seastar6futureINS_15semaphore_unitsINS_33named_semaphore_exception_factoryENSt3__16chrono12steady_clockEEEE4thenIZNS_14with_semaphoreIS2_ZN7cluster18controller_backend16reconcile_topicsEvE4$_21S5_EENS_8futurizeINS3_13invoke_resultIT0_JEE4typeEE4typeERNS_15basic_semaphoreIT_T1_EEmOSF_EUlSL_E_NS0_IvEEEESF_OSL_ENUlDpOT_E_clIJS6_EEEDaSV_ at controller_backend.cc:?
_ZN7seastar20noncopyable_functionIFNS_6futureIvEEONS_15semaphore_unitsINS_33named_semaphore_exception_factoryENSt3__16chrono12steady_clockEEEEE17direct_vtable_forIZNS1_IS8_E4thenIZNS_14with_semaphoreIS4_ZN7cluster18controller_backend16reconcile_topicsEvE4$_21S7_EENS_8futurizeINS5_13invoke_resultIT0_JEE4typeEE4typeERNS_15basic_semaphoreIT_T1_EEmOSL_EUlSR_E_S2_EESL_OSR_EUlDpOT_E_E4callEPKSB_S9_ at controller_backend.cc:?
seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>::operator()(seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) const at ??:?
decltype ((static_cast<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&>({parm#1}))(static_cast<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>>({parm#2}))) std::__1::__invoke<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at application.cc:?
std::__1::invoke_result<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::type std::__1::invoke<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at ??:?
auto seastar::internal::future_invoke<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at ??:?
seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&) const::{lambda()#1}::operator()() const at ??:?
void seastar::futurize<seastar::future<void> >::satisfy_with_result_of<seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&) const::{lambda()#1}>(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&) at ??:?
seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&) const at ??:?
seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::run_and_dispose() at ??:?
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at reactor.cc:?
seastar::reactor::run_some_tasks() at reactor.cc:?
seastar::reactor::do_run() at reactor.cc:?
seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91::operator()() const at reactor.cc:?
decltype ((static_cast<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91&>({parm#1}))()) std::__1::__invoke<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91&>(seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91&) at reactor.cc:?
void std::__1::__invoke_void_return_wrapper<void, true>::__call<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91&>(seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91&) at reactor.cc:?
std::__1::__function::__alloc_func<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91, std::__1::allocator<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91>, void ()>::operator()() at reactor.cc:?
std::__1::__function::__func<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91, std::__1::allocator<seastar::smp::configure(seastar::smp_options const&, seastar::reactor_options const&)::$_91>, void ()>::operator()() at reactor.cc:?
std::__1::__function::__value_func<void ()>::operator()() const at application.cc:?
std::__1::function<void ()>::operator()() const at application.cc:?
seastar::posix_thread::start_routine(void*) at posix.cc:?

NyaliaLui avatar Oct 11 '22 19:10 NyaliaLui

Another piece to the backtrace that I missed on docker-rp-11

MemcmpInterceptorCommon(void*, int (*)(void const*, void const*, unsigned long), void const*, void const*, unsigned long) at asan_interceptors.cpp:?
__interceptor_memcmp at ??:?
fmt::v8::detail::equal2(char const*, char const*) at application.cc:?
void fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref) at application.cc:?
fmt::v8::appender fmt::v8::vformat_to<fmt::v8::appender, 0>(fmt::v8::appender, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<fmt::v8::appender, char> >) at protobuf.cc:?
auto fmt::v8::formatter<wrapped_log_level, char, void>::format<fmt::v8::basic_format_context<fmt::v8::appender, char> >(wrapped_log_level, fmt::v8::basic_format_context<fmt::v8::appender, char>&) const at log.cc:?
void fmt::v8::detail::value<fmt::v8::basic_format_context<fmt::v8::appender, char> >::format_custom_arg<wrapped_log_level, fmt::v8::formatter<wrapped_log_level, char, void> >(void*, fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) at log.cc:?
fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle::format(fmt::v8::basic_format_parse_context<char, fmt::v8::detail::error_handler>&, fmt::v8::basic_format_context<fmt::v8::appender, char>&) const at application.cc:?
fmt::v8::detail::default_arg_formatter<char>::operator()(fmt::v8::basic_format_arg<fmt::v8::basic_format_context<fmt::v8::appender, char> >::handle) at application.cc:?
char const* fmt::v8::detail::parse_replacement_field<char, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&>(char const*, char const*, fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref)::format_handler&) at application.cc:?
void fmt::v8::detail::vformat_to<char>(fmt::v8::detail::buffer<char>&, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<std::__1::conditional<std::is_same<fmt::v8::type_identity<char>::type, char>::value, fmt::v8::appender, std::__1::back_insert_iterator<fmt::v8::detail::buffer<fmt::v8::type_identity<char>::type> > >::type, fmt::v8::type_identity<char>::type> >, fmt::v8::detail::locale_ref) at application.cc:?
seastar::internal::log_buf::inserter_iterator fmt::v8::vformat_to<seastar::internal::log_buf::inserter_iterator, 0>(seastar::internal::log_buf::inserter_iterator, fmt::v8::basic_string_view<char>, fmt::v8::basic_format_args<fmt::v8::basic_format_context<fmt::v8::appender, char> >) at application.cc:?
seastar::logger::do_log(seastar::log_level, seastar::logger::log_writer&) at log.cc:?
void seastar::logger::log<char const*, int, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&>(seastar::log_level, seastar::logger::format_info, char const*&&, int&&, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&) at ??:?
void seastar::logger::trace<char const*, int, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&>(seastar::logger::format_info, char const*&&, int&&, model::ntp const&, raft::group_configuration&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&) at ??:?
cluster::check_configuration_update(detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, seastar::lw_shared_ptr<cluster::partition> const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
seastar::future<std::__1::error_code> cluster::controller_backend::apply_configuration_change_on_leader<cluster::controller_backend::update_partition_replica_set(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >)::$_27>(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, cluster::controller_backend::update_partition_replica_set(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >)::$_27&&) at controller_backend.cc:?
cluster::controller_backend::update_partition_replica_set(model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
cluster::controller_backend::execute_reconfiguration(cluster::topic_table_delta::op_type, model::ntp const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
cluster::controller_backend::process_partition_reconfiguration(cluster::topic_table_delta::op_type, model::ntp, cluster::partition_assignment const&, std::__1::vector<model::broker_shard, std::__1::allocator<model::broker_shard> > const&, absl::lts_20220623::flat_hash_map<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> >, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >, absl::lts_20220623::hash_internal::Hash<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::equal_to<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > >, std::__1::allocator<std::__1::pair<detail::base_named_type<int, model::node_id_model_type, std::__1::integral_constant<bool, true> > const, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> > > > > const&, detail::base_named_type<long, model::revision_id_model_type, std::__1::integral_constant<bool, true> >) at ??:?
cluster::controller_backend::execute_partition_op(cluster::topic_table_delta const&) at ??:?
cluster::controller_backend::reconcile_ntp(std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> >&) at ??:?
cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}::operator()(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&) const at controller_backend.cc:?
seastar::future<void> seastar::futurize<seastar::future<void> >::invoke<cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&>(cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}&&, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&) at controller_backend.cc:?
auto seastar::futurize_invoke<cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&>(cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}&&, std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&) at controller_backend.cc:?
seastar::future<void> seastar::parallel_for_each<absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}>(absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, absl::lts_20220623::container_internal::raw_hash_set<absl::lts_20220623::container_internal::FlatHashMapPolicy<model::ntp, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >, absl::lts_20220623::hash_internal::Hash<model::ntp>, std::__1::equal_to<model::ntp>, std::__1::allocator<std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > > > >::iterator, cluster::controller_backend::reconcile_topics()::$_21::operator()() const::{lambda(std::__1::pair<model::ntp const, std::__1::vector<cluster::topic_table_delta, std::__1::allocator<cluster::topic_table_delta> > >&)#1}&&) at controller_backend.cc:?
cluster::controller_backend::reconcile_topics()::$_21::operator()() const at controller_backend.cc:?
seastar::future<void> seastar::futurize<seastar::future<void> >::invoke<cluster::controller_backend::reconcile_topics()::$_21>(cluster::controller_backend::reconcile_topics()::$_21&&) at controller_backend.cc:?
auto seastar::futurize_invoke<cluster::controller_backend::reconcile_topics()::$_21>(cluster::controller_backend::reconcile_topics()::$_21&&) at controller_backend.cc:?
auto seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}::operator()<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>) at controller_backend.cc:?
seastar::future<void> seastar::futurize<seastar::future<void> >::invoke<seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at controller_backend.cc:?
auto seastar::futurize_invoke<seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::with_semaphore<seastar::named_semaphore_exception_factory, cluster::controller_backend::reconcile_topics()::$_21, std::__1::chrono::steady_clock>(seastar::basic_semaphore<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&, unsigned long, cluster::controller_backend::reconcile_topics()::$_21&&)::{lambda(auto:1)#1}&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at controller_backend.cc:?
_ZZN7seastar6futureINS_15semaphore_unitsINS_33named_semaphore_exception_factoryENSt3__16chrono12steady_clockEEEE4thenIZNS_14with_semaphoreIS2_ZN7cluster18controller_backend16reconcile_topicsEvE4$_21S5_EENS_8futurizeINS3_13invoke_resultIT0_JEE4typeEE4typeERNS_15basic_semaphoreIT_T1_EEmOSF_EUlSL_E_NS0_IvEEEESF_OSL_ENUlDpOT_E_clIJS6_EEEDaSV_ at controller_backend.cc:?
_ZN7seastar20noncopyable_functionIFNS_6futureIvEEONS_15semaphore_unitsINS_33named_semaphore_exception_factoryENSt3__16chrono12steady_clockEEEEE17direct_vtable_forIZNS1_IS8_E4thenIZNS_14with_semaphoreIS4_ZN7cluster18controller_backend16reconcile_topicsEvE4$_21S7_EENS_8futurizeINS5_13invoke_resultIT0_JEE4typeEE4typeERNS_15basic_semaphoreIT_T1_EEmOSL_EUlSR_E_S2_EESL_OSR_EUlDpOT_E_E4callEPKSB_S9_ at controller_backend.cc:?
seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>::operator()(seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) const at ??:?
decltype ((static_cast<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&>({parm#1}))(static_cast<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>>({parm#2}))) std::__1::__invoke<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at application.cc:?
std::__1::invoke_result<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::type std::__1::invoke<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at ??:?
auto seastar::internal::future_invoke<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&) at ??:?
seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&) const::{lambda()#1}::operator()() const at ??:?
void seastar::futurize<seastar::future<void> >::satisfy_with_result_of<seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&) const::{lambda()#1}>(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&) at ??:?
seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&) const at ??:?
seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> (seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock>&&)>&, seastar::future_state<seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >&&)#1}, seastar::semaphore_units<seastar::named_semaphore_exception_factory, std::__1::chrono::steady_clock> >::run_and_dispose() at ??:?
seastar::reactor::run_tasks(seastar::reactor::task_queue&) at reactor.cc:?
seastar::reactor::run_some_tasks() at reactor.cc:?
seastar::reactor::do_run() at reactor.cc:?
seastar::reactor::run() at reactor.cc:?
seastar::app_template::run_deprecated(int, char**, std::__1::function<void ()>&&) at app-template.cc:?
seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&) at app-template.cc:?
application::run(int, char**) at ??:?
main at ??:?
?? ??:0
_start at ??:?

NyaliaLui avatar Oct 11 '22 19:10 NyaliaLui

I guess this is a reactor stall? and the actual Timeout might be unrelated to the trace.

dotnwat avatar Oct 12 '22 03:10 dotnwat

Yeah the last redpanda line in the backtrace is here: https://github.com/graphcareful/redpanda/blob/dev/src/v/cluster/controller_backend.cc#L169 upon the printing of a tracelog. Doesn't look like the call to logger contained a vector with many elements either.

graphcareful avatar Oct 18 '22 08:10 graphcareful

Took a second look, seems like the test decomissions nodes 3 and 4. Then waits until complete, then recommissions node 3. Finally the raft_learner_recovery_rate is shrunk and the test is supposed to assert that node 4 was removed from the cluster. This is where the test fails, the node is never removed and remains in the draining state up until the timeout expires.

Looking into the logs for node 4 I observe 6 reactor stalls, each occurring in an important place in controller_backend right at the time a log like this is printed:

TRACE 2022-10-11 16:42:28,697 [shard 1] cluster - controller_backend.cc:873 - [{kafka/__consumer_offsets/12}] processing reconfiguration update command with target replicas: {{node_id: 1, shard: 0}, {node_id: 2, shard: 1}, {node_id: 5, shard: 1}}, previous replica set: {{node_id: 2, shard: 1}, {node_id: 3, shard: 1}, {node_id: 5, shard: 1}}, revision: 87

It seems to me like the reactor stall prevented the cluster from executing the logic necessary to remove the node. Retries occur but more reactor stalls occur.

Anyone have an idea why so many reactor stalls are occurring within this test? Some sort of issue with the env?

graphcareful avatar Nov 09 '22 15:11 graphcareful

Reactor stalls on a debug build aren't necessarily surprising, but if it's happening on a particular test repeatedly that's probably a sign that there really is some pathological loop in there.

jcsp avatar Nov 11 '22 12:11 jcsp

True, I'm going to close this ticket as its seeming like there is an environment issue here. Out of 5 of the release builds two failed with failures in this unit test:

The following tests FAILED:

	 26 - storage_single_thread_rpunit (Failed)

and the rest failed to successfully run any ducktape tests, reporting Socket and PipeTimeout's for the entire duration. All of the debug builds have more then one reactor stall reported.

graphcareful avatar Nov 11 '22 14:11 graphcareful

https://buildkite.com/redpanda/redpanda/builds/18525#01846f78-61b3-42e0-97f8-dd8e7e48704b

also this one is in the debug build

andijcr avatar Nov 14 '22 15:11 andijcr

Re-opening, looks like the logs from Andrea's failures are different from all of the other failed runs.

graphcareful avatar Nov 14 '22 17:11 graphcareful

After looking at the logs, seems like yet another backtrace correlating to around the time i'd expect the system to be removing the decommissioned node , however it wasn't automatically decoded:

DEBUG 2022-11-13 06:58:08,870 [shard 1] raft - [group_id:13, {kafka/topic-weawzwqlfa/2}] consensus.cc:2614 - triggering leadership notification with term: 2, new leader: {nullopt}
Reactor stalled for 32 ms on shard 0. Backtrace: 0x3571849a 0x480351f4 0x480346cc 0x47e109e3 0x47e0ab54 0x47e0a5f7 0x47e0adb8 0x47e1082a 0x42abf 0x3576107a 0x356e1a82 0x356e18cc 0x356e1459 0x356e3707 0x3575f565 0x357bc3d6 0x357bc35c 0x35cdf8cc 0x35cdf86e 0x35cdf71e 0x35cdf3e5 0x35a27ac8 0x47988a78 0x479a27e2 0x479a2878 0x357bc569 0x379d7568 0x379d750c 0x379d7484 0x48ace1cc 0x48a769d7 0x48a7687f 0x48a7680f 0x48a7675d 0x48a766cb 0x35ca646b 0x35ca62f3 0x35ca621f 0x35ca6013 0x35ca5f5c 0x35ca5cec 0x35ca5ae6 0x35ca55fa 0x47e30b2d 0x47e3698e 0x47e3b4ef 0x47e39078 0x47b28a5c 0x47b26115 0x357dfa54 0x3579ce2f 0x2d58f 0x2d648 0x356dc964
...
...

graphcareful avatar Nov 14 '22 18:11 graphcareful

That last failure link (job 18525) is the one and only failure in the last 30 days in this test.

503s getting broker list happen when the node cannot get health information.

jcsp avatar Nov 25 '22 10:11 jcsp

This is a test bug. The test is sending /v1/brokers requests to any node, but that includes the decommissioned node. The decommissioned node will always respond with 503 because it cannot join controller raft group.

Other cases in this class pick a survivor node to use for admin API requests, this test should do the same.

jcsp avatar Nov 25 '22 11:11 jcsp