ATS Crash in state_read_server_response_header，7.1.x, 8.x etc...

[zhangqiang-01]

I use I used TSHttpIntercept to intercept http requests and turn to a custom state machine, in custom SM, when read event callback, close vc directly, ats will crash.

codes:

struct SMCtx {
  TSVConn net_vc;
  TSIOBuffer req_bufp;
  TSIOBufferReader req_readerp;
  // out
  TSIOBuffer bufferp;
  TSIOBufferReader readerp;
};

static int
test_sm(TSCont contp, TSEvent event, void *edata)
{
  SMCtx *ctx = (SMCtx *)TSContDataGet(contp);
  TSVConnClose(ctx->net_vc);
  TSContDestroy(contp);
  return TS_EVENT_NONE;
}

static int
test_accept_handle(TSCont contp, TSEvent event, void *edata)
{
  if (event == TS_EVENT_NET_ACCEPT_FAILED) {
    TSError("accept failed");	
  }else if (event == TS_EVENT_NET_ACCEPT) {
    TSError("accept success");
  }else{
    TSError("default, error!!!!!");
  }

  TSVConn vc = (TSVConn)edata;
  SMCtx *ctx = new SMCtx;
  ctx->req_bufp = TSIOBufferCreate();
  ctx->req_readerp = TSIOBufferReaderAlloc(ctx->req_bufp);
  ctx->net_vc = vc;
  TSCont _contp = TSContCreate(test_sm, TSMutexCreate());
  TSContDataSet(_contp, (void *)ctx);
  TSVConnRead(vc, _contp, ctx->req_bufp, INT64_MAX);
  TSContDestroy(contp);
  return 0;
}

void
test_handler(TSHttpTxn txnp)
{
  TSCont contp = TSContCreate(test_accept_handle, TSMutexCreate());
  TSHttpTxnIntercept(contp, txnp);
  return;
}

coredump

warning: Unexpected size of section `.reg-xstate/1271963' in core file.
#0  0x00005634598a5a90 in IOBufferBlock::end (this=0x0) at /data/bdcdn_open_soft/soft/source/trafficserver-7.1.8/iocore/eventsystem/I_IOBuffer.h:327
327	    return _end;
[Current thread is 1 (Thread 0x7f50223c6700 (LWP 1271963))]
(gdb) bt
#0  0x00005634598a5a90 in IOBufferBlock::end (this=0x0) at /data/bdcdn_open_soft/soft/source/trafficserver-7.1.8/iocore/eventsystem/I_IOBuffer.h:327
#1  0x0000563459a3f7df in HdrHeap::attach_block (this=0x7f4fd0036000, b=0x0, use_start=0x0) at HdrTSOnly.cc:173
#2  0x0000563459a3f50f in HTTPHdr::parse_resp (this=0x7f5004105298, parser=0x7f50041065d0, r=0x7f4fc014eb18, bytes_used=0x7f50223c5a04, eof=true) at HdrTSOnly.cc:111
#3  0x000056345995a467 in HttpSM::state_read_server_response_header (this=0x7f5004104a40, event=104, data=0x7f4fa0106b98) at HttpSM.cc:1884
#4  0x000056345995db96 in HttpSM::main_handler (this=0x7f5004104a40, event=104, data=0x7f4fa0106b98) at HttpSM.cc:2683
#5  0x000056345987a472 in Continuation::handleEvent (this=0x7f5004104a40, event=104, data=0x7f4fa0106b98) at /data/bdcdn_open_soft/soft/source/trafficserver-7.1.8/iocore/eventsystem/I_Continuation.h:152
#6  0x00005634598c5b0a in PluginVC::process_read_side (this=0x7f4fa0106a70, other_side_call=false) at PluginVC.cc:631
#7  0x00005634598c3e4d in PluginVC::main_handler (this=0x7f4fa0106a70, event=1, data=0x7f4f58106ec0) at PluginVC.cc:224
#8  0x000056345987a472 in Continuation::handleEvent (this=0x7f4fa0106a70, event=1, data=0x7f4f58106ec0) at /data/bdcdn_open_soft/soft/source/trafficserver-7.1.8/iocore/eventsystem/I_Continuation.h:152
#9  0x0000563459b4a036 in EThread::process_event (this=0x7f502938e010, e=0x7f4f58106ec0, calling_code=1) at UnixEThread.cc:140
#10 0x0000563459b4a280 in EThread::process_queue (this=0x7f502938e010, NegativeQueue=0x7f50223c5e40) at UnixEThread.cc:175
#11 0x0000563459b4a387 in EThread::execute_regular (this=0x7f502938e010) at UnixEThread.cc:207
#12 0x0000563459b4a680 in EThread::execute (this=0x7f502938e010) at UnixEThread.cc:278
#13 0x0000563459b49504 in spawn_thread_internal (a=0x56345a838b40) at Thread.cc:84
#14 0x00007f502c8f54a4 in start_thread (arg=0x7f50223c6700) at pthread_create.c:456
#15 0x00007f502bb96d0f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:97
(gdb) f 3
#3  0x000056345995a467 in HttpSM::state_read_server_response_header (this=0x7f5004104a40, event=104, data=0x7f4fa0106b98) at HttpSM.cc:1884
1884	    t_state.hdr_info.server_response.parse_resp(&http_parser, server_buffer_reader, &bytes_used, server_entry->eos);
(gdb) p vio
$1 = (VIO *) 0x7f4fa0106b98
(gdb) p *vio
$2 = {_cont = 0x7f5004104a40, nbytes = 9223372036854775807, ndone = 0, op = 1, buffer = {name = 0x0, mbuf = 0x7f4fc014eb00, entry = 0x0}, vc_server = 0x7f4fa0106a70, mutex = {m_ptr = 0x7f4ffc26cbd0}, _disabled = false}
(gdb) p server_buffer_reader
$3 = (IOBufferReader *) 0x7f4fc014eb18
(gdb) p *server_buffer_reader
$4 = {accessor = 0x0, mbuf = 0x7f4fc014eb00, block = {m_ptr = 0x0}, start_offset = 0, size_limit = 9223372036854775807}
(gdb) p *(server_buffer_reader->mbuf)
$5 = {size_index = 6, water_mark = 0, _writer = {m_ptr = 0x0}, readers = {{accessor = 0x0, mbuf = 0x7f4fc014eb00, block = {m_ptr = 0x0}, start_offset = 0, size_limit = 9223372036854775807}, {accessor = 0x0, mbuf = 0x0, block = {
        m_ptr = 0x0}, start_offset = 0, size_limit = 9223372036854775807}, {accessor = 0x0, mbuf = 0x0, block = {m_ptr = 0x0}, start_offset = 0, size_limit = 9223372036854775807}, {accessor = 0x0, mbuf = 0x0, block = {m_ptr = 0x0}, 
      start_offset = 0, size_limit = 9223372036854775807}, {accessor = 0x0, mbuf = 0x0, block = {m_ptr = 0x0}, start_offset = 0, size_limit = 9223372036854775807}}, _location = 0x563459b7d0e8 "memory/IOBuffer/HttpServerSession.cc:88"}
(gdb) 

After analyzing, I found that the buffer created by calling new_empty_MIOBuffer in HttpServerSession :: new_connection has no IOBufferBlock. But this situation is not compatible in state_read_server_response_header.

Compare 6.2.x and 7.x / 8.x In 6.2.x

int
HttpSM::state_read_server_response_header(int event, void *data)
{
 ...
  int bytes_used = 0;
  VIO *vio       = (VIO *)data;

  switch (event) {
  case VC_EVENT_EOS:
    server_entry->eos = true;

    // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

    // If no bytes were transmitted, the parser treats
    // as a good 0.9 response which is technically is
    // but it's indistinguishable from an overloaded
    // server closing the connection so don't accept
    // zero length responses
    if (vio->ndone == 0) {
      // Error handling function
      handle_server_setup_error(event, data);
      return 0;
    }
  // Fall through
  case VC_EVENT_READ_READY:
  case VC_EVENT_READ_COMPLETE:
    // More data to parse
    break;

  case VC_EVENT_ERROR:
  case VC_EVENT_INACTIVITY_TIMEOUT:
  case VC_EVENT_ACTIVE_TIMEOUT:
    // Error handling function
    handle_server_setup_error(event, data);
    return 0;
  }
  ...

in 7.x/8.x

int
HttpSM::state_read_server_response_header(int event, void *data)
{
  ....
  int bytes_used = 0;
  VIO *vio       = (VIO *)data;

  switch (event) {
  case VC_EVENT_EOS:
    server_entry->eos = true;

  // Fall through
  case VC_EVENT_READ_READY:
  case VC_EVENT_READ_COMPLETE:
    // More data to parse
    break;

  case VC_EVENT_ERROR:
  case VC_EVENT_INACTIVITY_TIMEOUT:
  case VC_EVENT_ACTIVE_TIMEOUT:
    // Error handling function
    handle_server_setup_error(event, data);
    return 0;
  }

[github-actions[bot]]

This issue has been automatically marked as stale because it has not had recent activity. Marking it stale to flag it for further consideration by the community.

[bryancall]

@SolidWallOfCode should this issue be close because of #3943?

@zhangqiang-01 are you seeing problems with the ATS 9 releases?