XTLS
splithttp使用nginx代理时GET请求会出现超时
完整性要求
- [X] 我保证阅读了文档，了解所有我编写的配置文件项的含义，而不是大量堆砌看似有用的选项或默认值。
- [X] 我提供了完整的配置文件和日志，而不是出于自己的判断只给出截取的部分。
- [X] 我搜索了 issues, 没有发现已提出的类似问题。
- [X] 问题在 Release 最新的版本上可以成功复现
描述
使用nginx代理的时候通过查看日志发现了存在两种情况的异常
1.同一个ID的数据先处理POST,后处理GET,处理GET时会出现超时,H2,H3都会出现
[28/Sep/2024:21:26:40 +0800] "POST /spre/f7fd9cea-56a8-4472-a793-498a57efcb77/0?x_padding=000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 HTTP/3.0" 200 0
[28/Sep/2024:21:27:40 +0800] "GET /spre/f7fd9cea-56a8-4472-a793-498a57efcb77?x_padding=0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 HTTP/3.0" 200 0 2024/09/28 21:27:40 [error] 720#720: *2214 upstream timed out (110: Connection timed out) while reading upstream, client:, request: "GET /spre/f7fd9cea-56a8-4472-a793-498a57efcb77?x_padding=0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 HTTP/3.0", upstream: "http://127.0.0.1:1234/spre/f7fd9cea-56a8-4472-a793-498a57efcb77?x_padding=0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"重现方式
持续运行xray，观看油管视频，大概五六分钟
客户端配置
{
"log": {
"access": "",
"error": "",
"loglevel": "warning"
},
"outbounds": [
{
"tag": "proxy",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
"users": [
{
"id": ""
}
]
}
]
},
"streamSettings": {
"network": "splithttp",
"security": "tls",
"tlsSettings": {
"allowInsecure": false,
"alpn": [
"h3"
],
"fingerprint": "chrome"
},
"splithttpSettings": {
"path": "spre",
"maxUploadSize": 1000000,
"maxConcurrentUploads": 10
}
},
"mux": {
"enabled": false,
"concurrency": -1
}
}
]
}服务端配置
nginx 配置 proxy_read_timeout 设置了30s,修改这个参数可以减少出现断流时的时间
user nginx;
worker_processes auto;
quic_bpf on;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$request_uri"';
access_log /var/log/nginx/access.log main;
map $http_upgrade $connection_upgrade {
default upgrade;
"" close;
}
map $remote_addr $proxy_forwarded_elem {
~^[0-9.]+$ "for=$remote_addr";
~^[0-9A-Fa-f:.]+$ "for=\"[$remote_addr]\"";
default "for=unknown";
}
map $http_forwarded $proxy_add_forwarded {
"~^(,[ \\t]*)*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*([ \\t]*,([ \\t]*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*)?)*$" "$http_forwarded, $proxy_forwarded_elem";
default "$proxy_forwarded_elem";
}
# server {
# listen 80;
# listen [::]:80;
# return 301 https://$host$request_uri;
# }
server {
listen 443 ssl default_server;
listen [::]:443 ssl default_server;
ssl_reject_handshake on;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_timeout 1h;
ssl_session_cache shared:SSL:10m;
}
server {
listen 443 quic reuseport;
listen 443 ssl reuseport;
listen [::]:443 quic reuseport;
listen [::]:443 ssl reuseport;
http2 on;
http3 on;
quic_gso on;
http3_stream_buffer_size 512k;
server_name exampl.com; # 填由 Nginx 加载的 SSL 证书中包含的域名，建议将域名指向服务端的 IP
ssl_certificate /etc/ssl/private/fullchain.cer;
ssl_certificate_key /etc/ssl/private/private.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_prefer_server_ciphers on;
ssl_stapling on;
ssl_stapling_verify on;
resolver 1.1.1.1 valid=60s;
resolver_timeout 2s;
client_header_timeout 1h;
keepalive_timeout 30m;
client_header_buffer_size 8k;
location /spre {
proxy_pass http://127.0.0.1:1234;
proxy_http_version 1.1;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 30s;
proxy_redirect off;
}
}
}
{
"log": {
"loglevel": "warning"
},
"inbounds": [
{
"listen": "127.0.0.1",
"port": "1234",
"protocol": "vless",
"settings": {
"clients": [
{
"id": ""
}
],
"decryption": "none"
},
"streamSettings": {
"network": "splithttp",
"splithttpSettings": {
"path": "/spre"
},
"sockopt": {
"tcpFastOpen": true,
"tcpMptcp": true,
"tcpNoDelay": true
}
},
"sniffing": {
"enabled": true,
"destOverride": [
"http",
"tls",
"quic"
]
}
}
]
}客户端日志
使用h3时会出现断流,h2未发现
2024/09/28 22:20:59 [Warning] [1204054532] app/proxyman/inbound: connection ends > proxy/http: connection ends > proxy/http: failed to write response > write tcp 127.0.0.1:10809->127.0.0.1:55692: wsasend: An established connection was aborted by the software in your host machine.
2024/09/28 22:21:00 from 127.0.0.1:55695 accepted //signaler-pa.youtube.com:443 [http -> proxy]
2024/09/28 22:21:00 [Warning] [966074724] app/proxyman/inbound: connection ends > proxy/http: connection ends > proxy/http: failed to write response > write tcp 127.0.0.1:10809->127.0.0.1:55693: wsasend: An established connection was aborted by the software in your host machine.
服务端日志
服务器端没有异常日志
GET方法超时的问题
https://github.com/XTLS/Xray-examples/blob/ca705fbd0ba4e302f60ea12a3aeff2c498c8d236/VLESS-WSS-Nginx/nginx.conf#L27
https://github.com/XTLS/Xray-examples/blob/ca705fbd0ba4e302f60ea12a3aeff2c498c8d236/VLESS-WSS-Nginx/nginx.conf#L27
这个GET方法不通的话当消耗完所有连接后,h3/h1就断流了，然后等待超时，配置5小时的话，那就要等待5小时才能恢复，这个问题不仅仅存在于nginx
感觉可能是因为下行不活跃导致的 GET 超时，但这个超时应该是正常的吧
这是nginx的access日志
这是nginx的error日志
nginx配置是这样的
location /spre {
proxy_pass http://127.0.0.1:1234;
proxy_http_version 1.1;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 30s;
proxy_redirect off;
proxy_buffer_size 16k;
proxy_buffers 4 32k;
proxy_busy_buffers_size 64k;
}
从结果来看应该是POST跟GET同时请求的，然后GET在30s之后超时才打印的日志
听起来怪怪的
这个断流的时候有测试过修改proxy_read_timeout 的值会影响恢复的时间，值越小恢复的越快
~~从字面意思上理解，这不应该是多久不活跃超时吗，咋变成断流冷却时间了~~
有一种可能是后方的 Xray 服务端和目标断开了连接，不再发数据给 Nginx 了，但没关闭到 Nginx 的 TCP，导致 Nginx 没关闭到 Xray 客户端的连接，Xray 客户端还以为连接正常？不过这个前置条件是怎么达成的，@mmmray 看下代码会不会这样
这倒是提醒了一件事，POST 到 Xray 服务端时如果出问题了，服务端应当把对应的 GET 也关掉 @mmmray
~~至于 CloseWrite CloseRead 什么的就先不考虑了~~
These kinds of bugs where xray would not close the transport properly if the wrapped VLESS stream has been closed (or the tunneled connection closed), they should've been fixed with https://github.com/XTLS/Xray-core/pull/3710
I also have a similar setup like in xray-examples and don't run into this kind of issue. I wonder if there is a difference between nginx vs haproxy vs caddy.
caddy
服务器端使用Xray-core v24.9.30版本
我参考xray的这个配置搭建了caddy，问题依旧存在 https://github.com/XTLS/Xray-examples/tree/main/VLESS-TLS-SplitHTTP-CaddyNginx
这是我的caddy配置
example.com {
tls /etc/ssl/private/fullchain.cer /etc/ssl/private/private.key
handle /split/* {
reverse_proxy http://127.0.0.1:1234
}
}
journalctl -fu caddy 查看caddy日志也是一直在输出类似下面的超时信息：
Oct 07 20:47:01 caddy[858257]: {"level":"error","ts":1728305221.0083828,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"127.0.0.1:1234","duration":0.002180221,"request":{"remote_ip":"...","remote_port":"32221","client_ip":"...","proto":"HTTP/2.0","method":"GET","host":"example.com:443","uri":"/split/847fc030-7330-46f4-aee4-7733d7321c05?x_padding=00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","headers":{"X-Forwarded-Host":["example.com:443"],"Accept-Encoding":["gzip"],"User-Agent":["Go-http-client/2.0"],"X-Forwarded-For":["..."],"X-Forwarded-Proto":["https"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"example.com"}},"error":"reading: context canceled"}
Oct 07 20:47:07 caddy[858257]: {"level":"error","ts":1728305227.806477,"logger":"http.handlers.reverse_proxy","msg":"aborting with incomplete response","upstream":"127.0.0.1:1234","duration":0.002216172,"request":{"remote_ip":"...","remote_port":"32221","client_ip":"...","proto":"HTTP/2.0","method":"GET","host":"example.com:443","uri":"/split/08a5c199-9643-4baa-b22e-6759b98b32f7?x_padding=00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","headers":{"User-Agent":["Go-http-client/2.0"],"Accept-Encoding":["gzip"],"X-Forwarded-For":["..."],"X-Forwarded-Proto":["https"],"X-Forwarded-Host":["example.com:443"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"example.com"}},"error":"reading: context canceled"}
我也碰到类似的问题。nginx1.27.1，xray24.9.30
我在nginx中添加了proxy_read_timeout 120s;后，nginx 的error.log中就不再出现upstream timed out (110: Connection timed out) while reading upstream, 问题了。
以下是我的nginx的配置
location /proxy {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://127.0.0.1:12578;
proxy_redirect off;
proxy_set_header Connection "";
proxy_http_version 1.1;
proxy_read_timeout 120s;
}
但是我的客户端中会出现 protocol error: received DATA after END_STREAM这样的问题。不确定是否和这个问题相关？
以下的客户端配置
"outbounds": [
{
"tag": "proxy",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
}
]
},
"streamSettings": {
"network": "splithttp",
"security": "tls",
"tlsSettings": {
"allowInsecure": false,
"serverName": "",
"alpn": [
"h3",
"h2",
"http/1.1"
],
"fingerprint": "random"
},
"splithttpSettings": {
"path": "/proxy",
"host": “",
"maxUploadSize": 1000000,
"maxConcurrentUploads": 10
}
},
"mux": {
"enabled": false,
"concurrency": -1
}
},
以下是服务端配置：
"inbounds": [
{
"port": 12578,
"protocol": "vless",
"settings": {
"clients": [
{
"id": "",
"level": 0
}
],
"decryption": "none",
"fallbacks": [
{
"dest": 443,
"xver": 1
}
]
},
"streamSettings": {
"network": "splithttp",
"security": "none",
"splithttpSettings": {
"path": "/proxy",
"host": ""
}
}
}
],
