bug: shutting down worker process don't disappear after apisix reload

[Lewisyixin]

Current Behavior

When a new deployed apisix reload, shutting down worker process will not disappear.

Expected Behavior

no shutting down worker process

Error Logs

No response

Steps to Reproduce

1. apisix reload
2. ps -ef|grep nginx

Environment

• APISIX version (run apisix version): 3.2.2
• Operating system (run uname -a): 3.10.0-514.26.2.el7.x86_64
• OpenResty / Nginx version (run openresty -V or nginx -V): 1.21.4.2 :

[Lewisyixin]

I have seen similar issue saying that this bug is fixed in 3.2TLS, but I can still reproduce in 3.2.2. https://github.com/apache/apisix/issues/10554

[Lewisyixin]

anyone can help?

[zhoujiexiong]

Hi @Lewisyixin

Does this issue recur stably on version 3.2.2? Is it convenient to provide the configuration of the recurrence problem and the related logs?

[Lewisyixin]

HI @zhoujiexiong this issue is easy to reproduce. I got a clean vm and just do 1. yum 3.2.2 version apisix 2. config etcd 3. init 4. start 5.reload and seems no error logs(all logs are here after I start and reload)

2024/08/19 15:37:25 [warn] 14425#14425: 1 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14426#14426: 2 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14431#14431: 4 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14428#14428: 5 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14427#14427: 3 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14431#14431: 4 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14426#14426: 2 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14428#14428: 5 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14425#14425: 1 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14426#14426: *21 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14431#14431: *34 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14428#14428: *53 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14425#14425: *75 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14427#14427: 3 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14427#14427: *92 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:30 [notice] 14506#14506: signal process started 2024/08/19 15:37:30 [emerg] 14513#14513: bind() to 127.0.0.1:9091 failed (98: Address already in use) 2024/08/19 15:37:30 [warn] 14508#14508: 452 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14507#14507: 453 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14509#14509: 454 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14510#14510: 455 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14508#14508: 452 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14507#14507: 453 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14508#14508: *474 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14507#14507: *495 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14510#14510: 455 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14509#14509: 454 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14510#14510: *532 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14509#14509: *549 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14513#14513: 584 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14513#14513: 584 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:31 [warn] 14513#14513: *598 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer

[Lewisyixin]

HI @zhoujiexiong any suggestions？ Do I have to upgrade my apisix version?

[zhoujiexiong]

HI @zhoujiexiong any suggestions？ Do I have to upgrade my apisix version?

Hi @Lewisyixin

Could you compare it with the latest version? :)

[Baoyuantop]

Hi @Lewisyixin, you may need to upgrade your version, there is currently no merged fix in version 3.2.2.