Jun Ouyang

Good catch, PR welcome.

PR welcome! Thanks

> IMO, fixing a buggy behavior is not a breaking change. It's arguable whether we should consider fixing a bug used as a feature in a special case. @oowl votes...

I have tried to place `set` command to the main block, but it did not resolve this problem. https://github.com/Kong/kong/pull/13201/commits/50d6c66e0b02e9c2fedf3ab54158a2a89f37db80 I think the problem is nginx can not get the location...

``` We also see worker.lua:164: failed to store event: queue overflow ``` That means some events were missing in some workers, So unhealthy events were not able to notify another...

> @oowl we are also sometimes seeing `2024/08/12 13:21:37 [error] 5315#0: *32717527 [lua] worker.lua:248: communicate(): event worker failed: failed to receive the first 2 bytes: closed, context: ngx.timer` is this...

> @oowl when we called curl localhost:8001/upstreams/printing-upgraded.v002/health?balancer_health=1, the response is `{"data":{"details":{"hosts":[{"host":"10.160.167.185","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.185","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.165","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.165","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.31","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.31","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.200","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.200","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.237","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.237","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.89","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.89","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.41","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.41","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.211","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.211","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.105","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.105","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.203","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.203","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.21","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.21","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.81","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.81","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.122","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.122","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.195","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.195","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.227","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.227","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.53","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.53","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.38","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.38","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.146","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.146","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.33","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.33","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.73","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.73","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.79","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.79","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.61","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.61","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.98","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.98","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.50","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.50","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.163","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.163","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.153","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.153","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.189","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.189","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.247","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.247","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.120","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.120","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.129","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.129","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.72","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.72","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.24","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.24","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.61","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.61","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.36","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.36","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.206","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.206","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.84","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.84","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.184","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.184","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.75","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.75","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.166.198","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.166.198","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.31","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.31","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.63","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.63","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.164.45","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.164.45","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.167.58","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.167.58","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768},{"host":"10.160.165.253","weight":{"available":100,"unavailable":0,"total":100},"addresses":[{"weight":100,"ip":"10.160.165.253","healthy":true,"port":32768}],"nodeWeight":100,"dns":"A","port":32768}],"healthy":true,"weight":{"available":4400,"unavailable":0,"total":4400}},"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3","health":"HEALTHY"},"node_id":"0dbceb58-8f05-496c-9c09-48fa16947ea3","next":null}` > > and when we call curl localhost:8001/upstreams/printing-upgraded.v002/targets we see `{"data":[{"id":"f466b583-e7ff-445e-8dea-083c79ba813d","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724218836.114,"target":"10.160.165.140:32770"},{"id":"0f1393e2-c4a7-4410-8da1-12956db77561","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724191836.659,"target":"10.160.164.105:32769"},{"id":"b7cad805-fc27-4055-afc0-0597f1c0189c","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724191236.387,"target":"10.160.166.35:32769"},{"id":"7706c4c7-42d9-44e0-addc-c38757461c87","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724190738.579,"target":"10.160.165.173:32769"},{"id":"84f9bf1f-7dea-4b77-9984-a0c0d40da96e","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724190216.961,"target":"10.160.164.136:32770"},{"id":"7db70aa4-8eaa-485d-abd1-fa48a3bd38f9","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724190216.785,"target":"10.160.167.80:32769"},{"id":"18c819ff-6158-4610-b817-7db69b932cef","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724189676.335,"target":"10.160.165.19:32770"},{"id":"999d013f-6e3e-4980-87bd-3fac91bb0efa","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724189676.157,"target":"10.160.167.107:32770"},{"id":"a71630cf-abfd-4c3a-a7e3-067628e03af2","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724189016.282,"target":"10.160.166.20:32769"},{"id":"5a7abdea-7a0b-4a5a-84a9-3cd339e6221e","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724188465.105,"target":"10.160.166.137:32770"},{"id":"4e2c1776-8618-42cd-8e5b-0d5953ea1908","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724187877.271,"target":"10.160.166.38:32768"},{"id":"78ce4630-175b-4d66-82e7-bec92c261404","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724187757.106,"target":"10.160.166.207:32769"},{"id":"54d6f98d-3d61-4396-aada-c484ff78967f","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724186607.07,"target":"10.160.164.9:32770"},{"id":"035c1cea-ccdf-4b04-8ec1-2f8deee02e9f","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724186076.654,"target":"10.160.164.72:32769"},{"id":"c7fcf2c2-4e85-479a-9b0c-eba1e504de57","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724181504.92,"target":"10.160.164.175:32768"},{"id":"92d760cd-a075-47c5-b306-7f1faca246a2","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724177363.394,"target":"10.160.165.5:32768"},{"id":"c776334b-f5d3-4810-a808-67c094560ed0","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724175697.106,"target":"10.160.165.66:32768"},{"id":"e8a977d3-f594-43cf-bace-deb7881c36fe","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724175637.074,"target":"10.160.167.66:32768"},{"id":"430475cc-5556-4356-934c-761b95810a28","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724175145.092,"target":"10.160.166.99:32768"},{"id":"ef1a251e-9026-46ce-a96a-df78dac81169","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724174557.164,"target":"10.160.167.40:32768"},{"id":"3c81a320-3bdd-4ffc-8859-83611edd3fa3","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724174078.544,"target":"10.160.166.30:32768"},{"id":"9b30953e-1ddb-4ec9-a635-57cb3ec66de4","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724173296.626,"target":"10.160.164.128:32768"},{"id":"65335009-67d5-4cc8-804b-2e59ce219e73","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724171660.221,"target":"10.160.164.58:32768"},{"id":"703ad0ae-6e59-4a04-a068-95077f6cd1a8","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724169396.924,"target":"10.160.164.236:32768"},{"id":"a9db7594-7003-4a5f-92b2-c0dc9a7c4381","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724168841.165,"target":"10.160.167.151:32768"},{"id":"758e8f7c-3d0c-409d-bc06-051caf249327","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724166636.474,"target":"10.160.166.138:32768"},{"id":"901b1dea-d8b9-4778-bae5-aa0e03b01b8f","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724164177.436,"target":"10.160.167.198:32768"},{"id":"713d2a46-8146-4266-85c7-557343236b87","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724159556.269,"target":"10.160.165.11:32768"},{"id":"7993ce39-15f0-432d-88bb-cd174b36a23f","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724155776.632,"target":"10.160.166.156:32769"},{"id":"70293a68-e568-4033-b4d5-0d878b59d4cc","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724105376.921,"target":"10.160.167.168:32769"},{"id":"19f56183-921f-40a3-b024-85d6a818076a","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724100277.11,"target":"10.160.165.184:32768"},{"id":"5b3d42d8-d835-4457-b194-bf4da7ab73c7","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724098416.459,"target":"10.160.165.215:32768"},{"id":"c565ab99-2edc-451b-b9e2-61d9d42ee79e","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724097876.364,"target":"10.160.164.182:32768"},{"id":"8363ed5a-cbdc-4cac-8ad3-f091760772f5","weight":100,"tags":null,"upstream":{"id":"0031d52b-eae2-41e5-bad8-55671fb2f9a3"},"created_at":1724088877.176,"target":"10.160.167.215:32769"}]` The IP is 10.160.164.33 is found in the...

> When will the worker system break, what is it constrained on? network/cpu/memory? maybe we can look at our container metrics based on your recommendation. that's the problem I want...

This error came from nginx event module https://github.com/nginx/nginx/blob/9c7a2c7ce4ad02a36df1bb0ee5e40a84610fffb9/src/event/ngx_event_connect.c#L242, openresty use `ngx_event_connect_peer` to connect ip:port, So it seems unable to avoid. Maybe openresty must describe this limitation in doc https://github.com/openresty/lua-nginx-module/blob/3f4005fa879fc2ed329275c47e238bff1f8b890e/src/ngx_http_lua_socket_tcp.c#L1442

Let's wait to merge https://github.com/Kong/lua-kong-nginx-module/pull/93 first.