warmcat
Multithreaded http server crashes under load
I've implemented simple multithreaded http server and built it under Windows OS with using pthreads4w library, the compiler is Visual Studio 2022 and build type is Release with Debug Information:
#include <libwebsockets.h>
#include <string.h>
#include <signal.h>
#include <string>
#include <vector>
#include <pthread.h>
#define COUNT_THREADS 4
static struct lws_context *context;
static volatile int interrupted;
void *thread_service(void *threadid)
{
while (lws_service_tsi(context, 10000,
(int)(lws_intptr_t)threadid) >= 0 &&
!interrupted)
;
pthread_exit(NULL);
return NULL;
}
void signal_cb(void *handle, int signum)
{
interrupted = 1;
switch (signum) {
case SIGTERM:
case SIGINT:
break;
default:
lwsl_err("%s: signal %d\n", __func__, signum);
break;
}
lws_context_destroy(context);
}
void sigint_handler(int sig)
{
signal_cb(NULL, sig);
}
static const char* html_content = "<!DOCTYPE html><html><body><h1>Hello world!</h1></body></html>\r\n";
extern "C" int lws_http_handler(struct lws* wsi, enum lws_callback_reasons reason, void* user, void* in, size_t len)
{
switch (reason)
{
case LWS_CALLBACK_HTTP: {
lws_callback_on_writable(wsi);
break;
}
case LWS_CALLBACK_HTTP_WRITEABLE: {
int status = 200;
std::string body = html_content;
std::string contentType = "text/html";
const static size_t reservedTitleSize = 512;
const auto bufferSize = LWS_PRE + reservedTitleSize + contentType.size() + body.length();
std::vector<unsigned char> buffer(bufferSize);
unsigned char* p = &buffer[LWS_PRE];
unsigned char* end = &buffer[bufferSize - LWS_PRE];
unsigned char* start = p;
// Add title and common headers (e.g., Content-Type)
if (lws_add_http_common_headers(
wsi, static_cast<int>(status), contentType.c_str(), body.size(), &p, end))
{
return 1;
}
// Finalize and send the HTTP header
if (lws_finalize_write_http_header(wsi, start, &p, end))
{
return 1;
}
// Send body
memcpy(p, body.c_str(), body.length());
if (lws_write(wsi, p, body.length(), LWS_WRITE_HTTP_FINAL) != static_cast<int>(body.length()))
{
return 1;
}
if (lws_http_transaction_completed(wsi))
{
return -1; // Close the connection due to lws internal error
}
break;
}
default:
break;
}
return 0;
}
static struct lws_protocols protocols[] = {
{"http", lws_http_handler, 0, 0},
LWS_PROTOCOL_LIST_TERM // Terminator
};
int main(int argc, const char **argv)
{
pthread_t pthread_service[COUNT_THREADS];
struct lws_context_creation_info info;
void *retval;
int n, logs = LLL_ERR | LLL_WARN
/* for LLL_ verbosity above NOTICE to be built into lws,
* lws must have been configured and built with
* -DCMAKE_BUILD_TYPE=DEBUG instead of =RELEASE */
/* | LLL_INFO */ /* | LLL_PARSER */ /* | LLL_HEADER */
/* | LLL_EXT */ /* | LLL_CLIENT */ /* | LLL_LATENCY */
/* | LLL_DEBUG */;
lws_set_log_level(logs, NULL);
memset(&info, 0, sizeof(info));
info.port = CONTEXT_PORT_NO_LISTEN;
info.options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS;
info.protocols = protocols;
info.gid = -1;
info.uid = -1;
info.user = &context;
info.count_threads = COUNT_THREADS;
info.timeout_secs = 500;
info.keepalive_timeout = 500;
info.connect_timeout_secs = 160;
info.timeout_secs_ah_idle = 150;
context = lws_create_context(&info);
if (!context) {
lwsl_err("lws init failed\n");
return 1;
}
info.iface = "0.0.0.0";
info.port = 8080;
if (lws_create_vhost(context, &info) == nullptr)
{
lwsl_err("lws vhost init failed\n");
return 1;
}
lwsl_notice(" Service threads: %d\n", lws_get_count_threads(context));
/* start all the service threads */
for (n = 0; n < lws_get_count_threads(context); n++)
if (pthread_create(&pthread_service[n], NULL, thread_service,
(void *)(lws_intptr_t)n))
lwsl_err("Failed to start service thread\n");
/* wait for all the service threads to exit */
while ((--n) >= 0)
pthread_join(pthread_service[n], &retval);
lwsl_notice("%s: calling external context destroy\n", __func__);
lws_context_destroy(context);
return 0;
}
I use the h2load as a multiple clients load and run h2load in loop for spawning and closing connections in script:
#!/bin/bash
# Check for sufficient arguments
if [ "$#" -lt 1 ]; then
echo "Usage: $0 <h2load_arguments>"
echo "Example: $0 http://10.211.55.9:8080/hello -n100000 -c100 -t16 --h1"
exit 1
fi
# Combine all passed arguments into the COMMAND
COMMAND="h2load $@"
echo "Starting endless h2load execution with: $COMMAND. Press Ctrl+C to stop."
# Infinite loop to execute the command
while true; do
$COMMAND
if [ $? -ne 0 ]; then
echo "Command execution failed. Exiting."
exit 1
fi
done
I run script with options:
./h2load.sh http://<server_ip>:8080/ -n10000 -c100 -t16 --h1
The script runs sometime with good rate but after several minutes it starts freezing sometimes cause of server. Finally server crashes at line https://github.com/warmcat/libwebsockets/blob/1fccae47ed9aac6c75f4dafb35693f7d90b5fd25/lib/plat/windows/windows-fds.c#L37 with error <path>\.conan2\p\b\libwe16e0083cfb612\b\src\lib\plat\windows\windows-fds.c:37: error: Debugger encountered an exception: Exception at 0x7ff7874e4973, code: 0xc0000005: read access violation at: 0x478, flags=0x0 (first chance).
May you please help me to resolve this issue?
Crash backtraces won't really help with that... if you want to help debug it, running it under linux + valgrind might provide useful information about what the exact code path that's wrong, eg, lock missing.
I've run this multithreaded http server under the valgrind and the CentOS Stream 9. It is built with GCC 11.5. No crash is observed in this case but it freezes forever and doesn't respond on any new request. Seems like there is dead lock in this case. valgrind_output.txt
Could you confirm this implementation of multithread http server is correct? Isn't there some implementation mistakes what leads to crash and dead lock? Or it is good implemenation and there is some issue in LWS?