cloud-controller-manager pod panic after deleting a svc with loadbalancer

Open benjaminhuo opened this issue 5 years ago • 1 comments
create a svc with loadbalancer type and specify qingcloud lb
the svc can be created and loadbalancer works fine
there're panic log like below in cloud-controller-manager pod after deleting the svc
I0721 05:35:03.915811       1 service_controller.go:729] Service has been deleted test-single-cluster/nginx-single-cluster. Attempting to cleanup load balancer resources
I0721 05:35:03.916266       1 event.go:258] Event(v1.ObjectReference{Kind:"Service", Namespace:"test-single-cluster", Name:"nginx-single-cluster", UID:"85a52f9e-94e8-4fcd-8da2-94a9b70a4ca6", APIVersion:"v1", ResourceVersion:"1689842", FieldPath:""}): type: 'Normal' reason: 'DeletingLoadBalancer' Deleting load balancer
I0721 05:35:26.795646       1 loadbalancer_impl.go:139] DeleteLoadBalancer takes total 22 seconds
E0721 05:35:26.795775       1 runtime.go:73] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference)
goroutine 242 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic(0x15d4360, 0x25b5890)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:69 +0x7b
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:51 +0x82
panic(0x15d4360, 0x25b5890)
	/usr/local/go/src/runtime/panic.go:969 +0x166
github.com/yunify/qingcloud-sdk-go/service.(*TagService).DetachTags(0x0, 0xc0002c7b40, 0x0, 0x1, 0x0)
	/Users/zheng1/work/go/pkg/mod/github.com/yunify/[email protected]/service/tag.go:272 +0x4e
github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor.DetachTagsFromResources(0x0, 0x0, 0x0, 0x0, 0xc000625ac0, 0x1, 0x1, 0x17eee03, 0x3, 0x0, ...)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor/tags.go:43 +0x12f
github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor.(*qingCloudLoadBalanceExecutor).Delete(0xc000014230, 0xc0005dbdd0, 0xb, 0x0, 0xc)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor/lb.go:305 +0x4b8
github.com/yunify/qingcloud-cloud-controller-manager/pkg/loadbalance.(*LoadBalancer).DeleteQingCloudLB(0xc0009b1380, 0x1a67540, 0xc000046048)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/loadbalance/loadbalancer.go:452 +0xf9
github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud.(*QingCloud).EnsureLoadBalancerDeleted(0xc000c02270, 0x1a67540, 0xc000046048, 0x17f68e5, 0xa, 0xc000441000, 0x0, 0x0)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud/loadbalancer_impl.go:142 +0xdb
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).processLoadBalancerDelete(0xc000616620, 0xc00000e4f0, 0xc000864810, 0x28, 0x1, 0x2d9)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:760 +0x141
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).processServiceDeletion(0xc000616620, 0xc000864810, 0x28, 0x4a, 0xc000bd5d48)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:750 +0x7a
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).syncService(0xc000616620, 0xc000864810, 0x28, 0x0, 0x0)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:730 +0x40a
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).processNextWorkItem(0xc000616620, 0x1a07a00)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:213 +0xf0
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).worker(0xc000616620)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:202 +0x2b
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00041bcc0)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5f
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00041bcc0, 0x3b9aca00, 0x0, 0x1, 0xc00008e720)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(0xc00041bcc0, 0x3b9aca00, 0xc00008e720)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d
created by k8s.io/kubernetes/pkg/controller/service.(*ServiceController).Run
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:191 +0x205
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x7c176e]
goroutine 242 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/runtime/runtime.go:58 +0x105
panic(0x15d4360, 0x25b5890)
	/usr/local/go/src/runtime/panic.go:969 +0x166
github.com/yunify/qingcloud-sdk-go/service.(*TagService).DetachTags(0x0, 0xc0002c7b40, 0x0, 0x1, 0x0)
	/Users/zheng1/work/go/pkg/mod/github.com/yunify/[email protected]/service/tag.go:272 +0x4e
github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor.DetachTagsFromResources(0x0, 0x0, 0x0, 0x0, 0xc000625ac0, 0x1, 0x1, 0x17eee03, 0x3, 0x0, ...)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor/tags.go:43 +0x12f
github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor.(*qingCloudLoadBalanceExecutor).Delete(0xc000014230, 0xc0005dbdd0, 0xb, 0x0, 0xc)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/executor/lb.go:305 +0x4b8
github.com/yunify/qingcloud-cloud-controller-manager/pkg/loadbalance.(*LoadBalancer).DeleteQingCloudLB(0xc0009b1380, 0x1a67540, 0xc000046048)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/loadbalance/loadbalancer.go:452 +0xf9
github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud.(*QingCloud).EnsureLoadBalancerDeleted(0xc000c02270, 0x1a67540, 0xc000046048, 0x17f68e5, 0xa, 0xc000441000, 0x0, 0x0)
	/Users/zheng1/work/go/src/github.com/yunify/qingcloud-cloud-controller-manager/pkg/qingcloud/loadbalancer_impl.go:142 +0xdb
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).processLoadBalancerDelete(0xc000616620, 0xc00000e4f0, 0xc000864810, 0x28, 0x1, 0x2d9)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:760 +0x141
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).processServiceDeletion(0xc000616620, 0xc000864810, 0x28, 0x4a, 0xc000bd5d48)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:750 +0x7a
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).syncService(0xc000616620, 0xc000864810, 0x28, 0x0, 0x0)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:730 +0x40a
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).processNextWorkItem(0xc000616620, 0x1a07a00)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:213 +0xf0
k8s.io/kubernetes/pkg/controller/service.(*ServiceController).worker(0xc000616620)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:202 +0x2b
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00041bcc0)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:152 +0x5f
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00041bcc0, 0x3b9aca00, 0x0, 0x1, 0xc00008e720)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(0xc00041bcc0, 0x3b9aca00, 0xc00008e720)
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88 +0x4d
created by k8s.io/kubernetes/pkg/controller/service.(*ServiceController).Run
	/Users/zheng1/work/go/pkg/mod/k8s.io/[email protected]/pkg/controller/service/service_controller.go:191 +0x
Jul 21 '20 05:07 benjaminhuo
/assign @zheng1 /assign @duanjiong
Jul 21 '20 05:07 benjaminhuo