cpp-httplib
cpp-httplib copied to clipboard
yhirose
Server-Sent Events Keep Socket in State CLOSE_WAIT After Client Leaves, Leading to Blocked Threads
Hi, first off: we really enjoy the library and it works really well. So thank you all! But we are facing the following issue:
Given you are using Firefox to connect to a server-sent event stream on a SSLServer. When closing the tab in Firefox and the stream does not send anymore data thereafter. Then the stream's socket stays in the state CLOSE_WAIT indefinitely.
After a few times the thread pool is exhausted and the server is unresponsive.
This is the minimal example code for the server:
#include <openssl/asn1.h>
#include <openssl/bio.h>
#include <openssl/err.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/x509.h>
#include <chrono>
#include <iostream>
#include <thread>
#define CPPHTTPLIB_OPENSSL_SUPPORT
#include "cpp-httplib/httplib.h"
std::unique_ptr<EVP_PKEY, void (*)(EVP_PKEY*)> create_private_key() {
std::unique_ptr<RSA, void (*)(RSA*)> rsa{RSA_new(), RSA_free};
std::unique_ptr<BIGNUM, void (*)(BIGNUM*)> bne{BN_new(), BN_free};
BN_set_word(bne.get(), RSA_F4);
RSA_generate_key_ex(rsa.get(), 4096, bne.get(), nullptr);
std::unique_ptr<EVP_PKEY, void (*)(EVP_PKEY*)> private_key{EVP_PKEY_new(), EVP_PKEY_free};
EVP_PKEY_assign_RSA(private_key.get(), rsa.release()) == 1;
return private_key;
}
std::unique_ptr<X509, void (*)(X509*)> create_certificate(EVP_PKEY* private_key) {
std::unique_ptr<X509, void (*)(X509*)> certificate{X509_new(), X509_free};
ASN1_INTEGER_set(X509_get_serialNumber(certificate.get()), 1);
X509_gmtime_adj(X509_get_notBefore(certificate.get()), 0);
X509_gmtime_adj(X509_get_notAfter(certificate.get()), 31536000L);
X509_set_pubkey(certificate.get(), private_key);
X509_sign(certificate.get(), private_key, EVP_sha256());
return certificate;
}
int main() {
using namespace std::chrono_literals;
OpenSSL_add_all_digests();
auto private_key = create_private_key();
auto certificate = create_certificate(private_key.get());
httplib::SSLServer ssl_server_{certificate.get(), private_key.get()};
SSL_CTX_set_min_proto_version(ssl_server_.ssl_context(), TLS1_2_VERSION);
SSL_CTX_set_max_proto_version(ssl_server_.ssl_context(), TLS1_2_VERSION);
SSL_CTX_set_verify(ssl_server_.ssl_context(), SSL_VERIFY_NONE, nullptr);
ssl_server_.Get("/silent", [](const httplib::Request& http_request, httplib::Response& http_response) {
http_response.set_chunked_content_provider(
"text/event-stream",
[](size_t offset, httplib::DataSink& sink) mutable {
std::this_thread::sleep_for(2s);
return true;
},
[](bool success) {});
});
std::thread([&]() {
ssl_server_.listen(std::string(), 443, AI_PASSIVE); // AI_PASSIVE: fill in my IP for me */
}).detach();
while (true) {
std::this_thread::sleep_for(120s);
}
return 0;
}
In the following I try to summarize my findings:
I modified the httplib.h file for better strace logging:
inline bool is_socket_alive(socket_t sock) {
const auto val = detail::select_read(sock, 0, 0);
if (val == 0) {
return true;
} else if (val < 0 && errno == EBADF) {
return false;
}
char buf[2024]; // <-- I changed the buffer from size 1 to size 2024
return detail::read_socket(sock, &buf[0], sizeof(buf), MSG_PEEK) > 0;
}
When I first connect to the running server using Firefox via https://10.149.108.178/silent:
netstat -untaten
tcp 0 0 10.149.108.178:443 10.254.10.125:35526 ESTABLISHED
and strace repeats this pattern:
strace -tt -ff -yy -xx -vv ./server_ssl
[pid 18699] 08:50:02.407287 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=2, tv_nsec=0}, 0xffff9cecc978) = 0
[pid 18699] 08:50:04.407685 pselect6(6, NULL, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, {tv_sec=5, tv_nsec=0}, NULL) = 1 (out [5], left {tv_sec=4, tv_nsec=999988875})
[pid 18699] 08:50:04.408020 pselect6(6, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, NULL, {tv_sec=0, tv_nsec=0}, NULL) = 0 (Timeout)
[pid 18699] 08:50:04.408164 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=2, tv_nsec=0}, 0xffff9cecc978) = 0
[pid 18699] 08:50:06.408569 pselect6(6, NULL, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, {tv_sec=5, tv_nsec=0}, NULL) = 1 (out [5], left {tv_sec=4, tv_nsec=999987875})
[pid 18699] 08:50:06.408911 pselect6(6, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, NULL, {tv_sec=0, tv_nsec=0}, NULL) = 0 (Timeout)
[pid 18699] 08:50:06.409064 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=2, tv_nsec=0}, ^C
Closing the tab in Firefox gives me:
netstat -untaten
tcp 32 0 10.149.108.178:443 10.254.10.125:35526 CLOSE_WAIT
and strace repeats this pattern:
strace -tt -ff -yy -xx -vv ./server_ssl
[pid 18699] 08:52:46.486016 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=2, tv_nsec=0}, 0xffff9cecc978) = 0
[pid 18699] 08:52:48.486560 pselect6(6, NULL, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, {tv_sec=5, tv_nsec=0}, NULL) = 1 (out [5], left {tv_sec=4, tv_nsec=999987875})
[pid 18699] 08:52:48.486898 pselect6(6, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, NULL, {tv_sec=0, tv_nsec=0}, NULL) = 1 (in [5], left {tv_sec=0, tv_nsec=0})
[pid 18699] 08:52:48.487056 recvfrom(5<TCP:[10.149.108.178:443->10.254.10.125:35526]>, "\x15\x03\x03\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x02\x71\xf3\xa9\xb5\xe2\x55\xcf\x86\x58\x59\xb3\xc2\x5c\x3e\x43\x4b\xce\x09", 2024, MSG_PEEK, NULL, NULL) = 31
[pid 18699] 08:52:48.487239 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=2, tv_nsec=0}, 0xffff9cecc978) = 0
[pid 18699] 08:52:50.487674 pselect6(6, NULL, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, {tv_sec=5, tv_nsec=0}, NULL) = 1 (out [5], left {tv_sec=4, tv_nsec=999987625})
[pid 18699] 08:52:50.488015 pselect6(6, [5<TCP:[10.149.108.178:443->10.254.10.125:35526]>], NULL, NULL, {tv_sec=0, tv_nsec=0}, NULL) = 1 (in [5], left {tv_sec=0, tv_nsec=0})
[pid 18699] 08:52:50.488171 recvfrom(5<TCP:[10.149.108.178:443->10.254.10.125:35526]>, "\x15\x03\x03\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x02\x71\xf3\xa9\xb5\xe2\x55\xcf\x86\x58\x59\xb3\xc2\x5c\x3e\x43\x4b\xce\x09", 2024, MSG_PEEK, NULL, NULL) = 31
[pid 18699] 08:52:50.488353 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=2, tv_nsec=0}, ^C
The decrypted Wireshark logs shows:
and the message the strace shows in
[pid 18699] 08:52:48.487056 recvfrom(5<TCP:[10.149.108.178:443->10.254.10.125:35526]>, "\x15\x03\x03\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x02\x71\xf3\xa9\xb5\xe2\x55\xcf\x86\x58\x59\xb3\xc2\x5c\x3e\x43\x4b\xce\x09", 2024, MSG_PEEK, NULL, NULL) = 31
Is the close_notify from the wireshark log.
Here I have the decrypted Wireshark log for a curl equivalent:
curl --insecure -v -i -X GET https://10.149.108.178:443/silent
Let me know if you need any other information. Thank you!
Ah, great news! I can trigger it with the library itself:
Sending one message from the server:
ssl_server_.Get("/silent", [](const httplib::Request& http_request, httplib::Response& http_response) {
http_response.set_chunked_content_provider(
"text/event-stream",
[initial_message_sent = false](size_t offset, httplib::DataSink& sink) mutable {
if (!initial_message_sent) {
std::string data = "a";
sink.write(data.c_str(), data.size());
initial_message_sent = true;
}
std::this_thread::sleep_for(2s);
return true;
},
[](bool success) {});
});
And the client sends a message in reply to the server's event:
#include <chrono>
#include <iostream>
#include <thread>
#define CPPHTTPLIB_OPENSSL_SUPPORT
#include "cpp-httplib/httplib.h"
int main() {
using namespace std::chrono_literals;
httplib::SSLClient client{"10.149.108.178"};
client.enable_server_certificate_verification(false);
client.Get(
"/silent",
httplib::Headers{},
[&](const httplib::Response& response) { return true; },
[&](const char* data, size_t data_length) {
if (client.is_socket_open()) {
std::string data = "asdf";
std::ignore = write(client.socket(), data.c_str(), data.size());
}
return true;
});
while (true) {
std::this_thread::sleep_for(120s);
}
return 0;
}
In my experience, closing an event-stream requires a server-side call to sink.done(), which you can try
@TimonNoethlichs I don't fully understand what you are trying to do, but my sample SSE server code might be helpful. https://github.com/yhirose/cpp-httplib/blob/52a18c78a52b1bcffc10506fe5fdc76568b3c646/example/ssesvr.cc#L71-L87
