brod
brod copied to clipboard
Published
20 hours ago •
kafka4beam
kafka4beam
Producer 'function clause' error
After producing 31 millions data I see following errors for all producers in logs:
2020-09-23 11:19:01.968201+00:00 [error] ** Generic server <0.629.0> terminating
** Last message in was {msg,<0.24609.0>,
{kpro_rsp,#Ref<0.3617632699.303562781.156674>,
produce,5,
#{responses =>
[#{partition_responses =>
[#{base_offset => 177509,
error_code => no_error,
log_append_time => -1,
log_start_offset => 0,
partition => 106}],
topic =>
<<"topic-name">>}],
throttle_time_ms => 0}}}
** When Server state == {state,<0.481.0>,<<"topic-name">>,106,
<0.24609.0>,#Ref<0.3617632699.300154889.31499>,
{buf,1000,1,10485760,3,0,0,
#Fun<brod_producer.1.19472292>,37,0,
{[],[]},
{[{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664738,1}],
[{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664667,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664667,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664667,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664672,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664672,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664672,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664738,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,
680,1,1600858664738,1}]},
[]},
500,#Ref<0.3617632699.304873505.192841>,undefined,
{resolved,5}}
** Reason for termination ==
** {function_clause,
[{brod_producer_buffer,ack,
[{buf,1000,1,10485760,3,0,0,#Fun<brod_producer.1.19472292>,37,0,
{[],[]},
{[{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600859941968,0},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664738,1}],
[{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664664,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664667,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664667,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664667,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664672,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664672,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664672,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664738,1},
{req,#Fun<brod_producer.2.19472292>,
#Fun<brod_producer_buffer.0.67628820>,680,1,
1600858664738,1}]},
[]},
#Ref<0.3617632699.303562781.156674>,177509],
[{file,"src/brod_producer_buffer.erl"},{line,146}]},
{brod_producer,handle_info,2,
[{file,"src/brod_producer.erl"},{line,393}]},
{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,637}]},
{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,711}]},
{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}
Sounds like producers got ack for some data that they don't have. Am I right?
My producer options:
{required_acks, 1},
{ack_timeout, 10s},
{partition_buffer_limit, 1000},
{partition_onwire_limit, 1},
{max_batch_size, 10 MB},
{max_retries, 3},
{max_linger_ms, 0},
{max_linger_count, 0},
{compression, snappy
Looks to me like the function_clause is caused by the onwire field being empty. I assume that means the ACK is either premature or stale (old). I'm afraid I don't know this code well enough to speculate on possible reasons for that.
