http server chunk error

[xiangjs6]

• My goal is: deal http chunk doubt
• My actions were: call mg_ http_ delete_ Chunks are only in the first chunk
• My expectation was: mongoose should not mark all subsequent chunks as delete
• The result I saw: after the last chunk, if the upper layer does not call delete, mongoose deletes the http request, resulting in subsequent http_Parse reported an error and closed the connection
• My question is: whether the mongoose behavior is correct, and whether the upper layer code should not handle it this way

Environment

• mongoose version: X.Y
• Compiler/IDE and SDK: gcc version 12.2.0

example

static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
  struct mg_http_message *hm = ev_data;
  if (ev == MG_EV_HTTP_CHUNK) {
      if (c->label[0] == 0) {
        mg_http_delete_chunk(c, hm);
        c->label[0] = 1;
      }
      if (hm->chunk.len == 0) {
        mg_http_reply(c, 200, "", "");
        c->label[0] = 0;
      }
  } else if (ev == MG_EV_HTTP_MSG) {
    mg_http_reply(c, 200, "", "");
  }
}

[cpq]

You cannot delete only one chunk. Either all chunks must be deleted, or none.

[xiangjs6]

You cannot delete only one chunk. Either all chunks must be deleted, or none.

If the upper layer can't process chunk in time, hoping that chunk data can be retained in mongoose's recv for a period of time, can it be done?

[cpq]

Every time a new piece of data arrives, it is reported as HTTP chunk, and appended to the c->recv. If you do not delete the chunk, then a new chunk is going to be appended to the non-deleted chunk. Yes you can delete non-deleted chunks later. I think if you set c->recv.len = hm->head.len; it'll delete all chunks and leave only request headers.

[xiangjs6]

Every time a new piece of data arrives, it is reported as HTTP chunk, and appended to the c->recv. If you do not delete the chunk, then a new chunk is going to be appended to the non-deleted chunk. Yes you can delete non-deleted chunks later. I think if you set c->recv.len = hm->head.len; it'll delete all chunks and leave only request headers.

If the last chunk arrives and the upper layer cannot process the chunk at this time, the chunk is not deleted. mongoose will delete the http header and report an error because it cannot parse the undeleted chunk behind the header, and disconnect the connection

[cpq]

I don't understand to be honest, what is your problem. Can you create a failing unit test that demonstrates your issue?

[xiangjs6]

I don't understand to be honest, what is your problem. Can you create a failing unit test that demonstrates your issue?

This is an example. In fact, handler_chunk will be executed in another thread

example

http server

struct connection_ctx {
  bool follow_message;
  bool fully_recv;
  int index;
};

struct mg_connection *queue[1024];
int q_start, q_end;

static void handler_chunk(void) {
  struct mg_http_message hm;
  int n;
  struct mg_connection *c;
  struct connection_ctx *conn_ctx;
  for (int i = q_start; i != q_end;
       i = (i + 1) % (sizeof(queue) / sizeof(*queue))) {
    c = queue[i];
    if (c == NULL) {
      continue;
    }
    conn_ctx = c->label;
    n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
    if (n < 0) {
      c->is_closing = 1;
      queue[i] = NULL;
      if (q_start == i) {
        do {
          q_start = (q_start + 1) % (sizeof(queue) / sizeof(*queue));
          if (queue[q_start] != NULL) {
            break;
          }
        } while (q_start != q_end);
      }
    }
    hm.chunk = mg_str_n((char *) &c->recv.buf[n], c->recv.len - n);
    // todo someshing
    if (conn_ctx->fully_recv) {
      mg_iobuf_del(&c->recv, 0, hm.message.len);
      conn_ctx->fully_recv = false;
      conn_ctx->follow_message = false;
      mg_http_reply(c, 200, "", "");
      queue[i] = NULL;
      if (q_start == i) {
        do {
          q_start = (q_start + 1) % (sizeof(queue) / sizeof(*queue));
          if (queue[q_start] != NULL) {
            break;
          }
        } while (q_start != q_end);
      }
    } else {
      mg_http_delete_chunk(c, &hm);
    }
  }
}

static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
  if (ev == MG_EV_POLL) {
    handler_chunk();
  } else if (ev == MG_EV_HTTP_CHUNK) {
    struct mg_http_message *hm = ev_data;
    struct connection_ctx *conn_ctx = c->label;
    if (conn_ctx->follow_message == false) {
      conn_ctx->follow_message = true;
      queue[q_end] = c;
      conn_ctx->index = q_end;
      q_end = (q_end + 1) % (sizeof(queue) / sizeof(*queue));
    } else if (hm->chunk.len == 0) {
      conn_ctx->fully_recv = true;
    }
  } else if (ev == MG_EV_HTTP_MSG) {
    struct mg_http_message *hm = ev_data;
    struct connection_ctx *conn_ctx = c->label;
    if (conn_ctx->follow_message == true) {
      conn_ctx->fully_recv = true;
      return;
    }
    mg_http_reply(c, 200, "", "");
  } else if (ev == MG_EV_ERROR) {
    struct connection_ctx *conn_ctx = c->label;
    if (conn_ctx->follow_message) {
      queue[conn_ctx->index] = NULL;
      if (q_start == conn_ctx->index) {
        do {
          q_start = (q_start + 1) % (sizeof(queue) / sizeof(*queue));
          if (queue[q_start] != NULL) {
            break;
          }
        } while (q_start != q_end);
      }
    }
  }
  (void) fn_data;
}

http client

I use python 3.10.6

import requests
client=requests.session()
resp=client.put("http://0.0.0.0:8000", data=bytes(100*1024*1024))

[cpq]

@xiangjs6 sorry, this code is too cryptic even to start looking into it. To avoid wasting yours and ours time, please create a MINIMAL failing unit test, as I suggested - or just close this issue.

[xiangjs6]

@xiangjs6 sorry, this code is too cryptic even to start looking into it. To avoid wasting yours and ours time, please create a MINIMAL failing unit test, as I suggested - or just close this issue.

I'm very sorry that my poor expression wasted your time. Thank you for your patient answer. The following code is my simplified code. I hope I can let you know what I mean.

struct connection_ctx {
  bool follow_message;
  bool fully_recv;
};
struct mg_connection *pending_conn;
static void handler_chunk(void) {
  struct mg_http_message hm;
  int n;
  struct mg_connection *c = pending_conn;
  struct connection_ctx *conn_ctx;
  if (c != NULL) {
    conn_ctx = c->label;
    n = mg_http_parse((char *) c->recv.buf, c->recv.len, &hm);
    hm.chunk = mg_str_n((char *) &c->recv.buf[n], c->recv.len - n);
    // todo write date to file
    if (conn_ctx->fully_recv) {
      mg_iobuf_del(&c->recv, 0, hm.message.len);
      mg_http_reply(c, 200, "", "");
      c->is_draining = 1;
      pending_conn = NULL;
    } else {
      mg_http_delete_chunk(c, &hm);
    }
  }
}

static void cb(struct mg_connection *c, int ev, void *ev_data, void *fn_data) {
  if (ev == MG_EV_POLL) {
    //  In fact, handler_chunk function is executed in other threads 
    handler_chunk();
  } else if (ev == MG_EV_HTTP_CHUNK) {
    struct mg_http_message *hm = ev_data;
    struct connection_ctx *conn_ctx = c->label;
    if (conn_ctx->follow_message == false) {
      if (pending_conn != false) {
          c->is_closing = 1;
      }
      pending_conn = c;
      conn_ctx->follow_message = true;
    } else if (hm->chunk.len == 0) {
      conn_ctx->fully_recv = true;
    }
  }
}

[cpq]

Thanks. I looked over your code, but still can't figure out what are you trying to do, and what is the problem.

I suggest to either NOT processing any chunks at all, or handle MG_EV_READ directly.