fluid icon indicating copy to clipboard operation
fluid copied to clipboard
verified publisher icon fluid-cloudnative

[BUG] Add timeout to `mount` and `umount` operation of csi plugin

Open cheyang opened this issue 4 years ago • 1 comments

What is your environment(Kubernetes version, Fluid version, etc.)

Describe the bug

Add timeout for mount and umount, when mountpoint is no responsive. we should avoid it hang.

I0611 05:41:17.627618    3789 nodeserver.go:140] /bin/umount /var/lib/kubelet/pods/b56fecf2-dcbb-4111-8b68-5ba0d3d89167/volumes/kubernetes.io~csi/default-jdfs-parallel-train/mount
I0611 05:41:17.631945    3789 nodeserver.go:143] fork/exec /bin/umount: resource temporarily unavailable
I0611 05:41:17.632712    3789 nodeserver.go:145] 
I0611 05:41:57.595940    3789 utils.go:97] GRPC call: /csi.v1.Node/NodeUnpublishVolume
I0611 05:41:57.595965    3789 utils.go:98] GRPC request: {"target_path":"/var/lib/kubelet/pods/505a5237-7155-4c81-9751-9fde0ec09066/volumes/kubernetes.io~csi/default-jdfs-parallel-train/mount","volume_id":"default-jdfs-parallel-train"}
I0611 05:41:57.596461    3789 nodeserver.go:140] /bin/umount /var/lib/kubelet/pods/505a5237-7155-4c81-9751-9fde0ec09066/volumes/kubernetes.io~csi/default-jdfs-parallel-train/mount
I0611 05:41:57.599992    3789 nodeserver.go:143] fork/exec /bin/umount: resource temporarily unavailable
I0611 05:41:57.600003    3789 nodeserver.go:145] 
runtime: failed to create new OS thread (have 8212 already; errno=11)
runtime: may need to increase max user processes (ulimit -u)
fatal error: newosproc

runtime stack:
runtime.throw(0x147e6f5, 0x9)
	/usr/local/go/src/runtime/panic.go:1116 +0x72
runtime.newosproc(0xc04f4c2a80)
	/usr/local/go/src/runtime/os_linux.go:161 +0x1ba
runtime.newm1(0xc04f4c2a80)
	/usr/local/go/src/runtime/proc.go:1753 +0xdc
runtime.newm(0x1526fa8, 0xc0000b4000)
	/usr/local/go/src/runtime/proc.go:1732 +0x8f
runtime.startm(0x0, 0xc000367501)
	/usr/local/go/src/runtime/proc.go:1869 +0x12a
runtime.wakep(...)
	/usr/local/go/src/runtime/proc.go:1953
runtime.resetspinning()
	/usr/local/go/src/runtime/proc.go:2415 +0x93
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:2527 +0x2de
runtime.park_m(0xc000001c80)
	/usr/local/go/src/runtime/proc.go:2690 +0x9d
runtime.mcall(0x0)
	/usr/local/go/src/runtime/asm_amd64.s:318 +0x5b

goroutine 1 [semacquire, 10786 minutes]:
sync.runtime_Semacquire(0xc000408988)
	/usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc000408980)
	/usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.(*nonBlockingGRPCServer).Wait(0xc000408980)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go:62 +0x2d
github.com/fluid-cloudnative/fluid/pkg/csi/fuse.(*driver).Run(0xc000424c00)
	/go/src/github.com/fluid-cloudnative/fluid/pkg/csi/fuse/driver.go:103 +0x1f6
main.handle()
	/go/src/github.com/fluid-cloudnative/fluid/cmd/csi/main.go:91 +0x64
main.glob..func1(0x21b60c0, 0xc000424ae0, 0x0, 0x3)
	/go/src/github.com/fluid-cloudnative/fluid/cmd/csi/main.go:43 +0x20
github.com/fluid-cloudnative/fluid/vendor/github.com/spf13/cobra.(*Command).execute(0x21b60c0, 0xc000424ab0, 0x3, 0x3, 0x21b60c0, 0xc000424ab0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/spf13/cobra/command.go:830 +0x29d
github.com/fluid-cloudnative/fluid/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x21b5e40, 0xc0000e6180, 0x0, 0xc0000f0058)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/spf13/cobra/command.go:914 +0x2fb
github.com/fluid-cloudnative/fluid/vendor/github.com/spf13/cobra.(*Command).Execute(...)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/spf13/cobra/command.go:864
main.main()
	/go/src/github.com/fluid-cloudnative/fluid/cmd/csi/main.go:80 +0x57

goroutine 6 [chan receive, 2 minutes]:
github.com/fluid-cloudnative/fluid/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x21cb3a0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/golang/glog/glog.go:882 +0x8b
created by github.com/fluid-cloudnative/fluid/vendor/github.com/golang/glog.init.0
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/golang/glog/glog.go:410 +0x26f

goroutine 17 [chan receive, 2 minutes]:
github.com/fluid-cloudnative/fluid/vendor/k8s.io/klog.(*loggingT).flushDaemon(0x21cb540)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/k8s.io/klog/klog.go:1010 +0x8b
created by github.com/fluid-cloudnative/fluid/vendor/k8s.io/klog.init.0
	/go/src/github.com/fluid-cloudnative/fluid/vendor/k8s.io/klog/klog.go:411 +0xd6

goroutine 46 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7fc98a11cee8, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc000434198, 0x72, 0x0, 0x0, 0x147bc32)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000434180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4
net.(*netFD).accept(0xc000434180, 0xc03b424df0, 0x0, 0x0)
	/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*UnixListener).accept(0xc000424c90, 0xc00010dd60, 0xc00010dd68, 0x18)
	/usr/local/go/src/net/unixsock_posix.go:162 +0x32
net.(*UnixListener).Accept(0xc000424c90, 0x1525de8, 0xc000450180, 0x1685440, 0xc03b424df0)
	/usr/local/go/src/net/unixsock.go:260 +0x64
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).Serve(0xc000450180, 0x1673620, 0xc000424c90, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:597 +0x210
github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.(*nonBlockingGRPCServer).serve(0xc000408980, 0x7fff0f306d53, 0x3e, 0x1666ae0, 0xc0004100c8, 0x168a920, 0xc0004100d8, 0x1685140, 0xc0004089a0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go:110 +0x323
created by github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.(*nonBlockingGRPCServer).Start
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go:56 +0xbe

goroutine 2946 [syscall, 8092 minutes]:
syscall.Syscall6(0xf7, 0x1, 0x2daa87, 0xc00060f728, 0x1000004, 0x0, 0x0, 0x4d1d01, 0xc000c71020, 0xc00060f768)
	/usr/local/go/src/syscall/asm_linux_amd64.s:41 +0x5
os.(*Process).blockUntilWaitable(0xc000816a50, 0x203000, 0x84f96f, 0x21cb3b8)
	/usr/local/go/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc000816a50, 0x1526c70, 0x1526c78, 0x1526c68)
	/usr/local/go/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
	/usr/local/go/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc00019c9a0, 0x0, 0x0)
	/usr/local/go/src/os/exec/exec.go:507 +0x60
os/exec.(*Cmd).Run(0xc00019c9a0, 0xc000a1a690, 0x2136cdd)
	/usr/local/go/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc00019c9a0, 0x0, 0xc00060f9e8, 0x1, 0x1, 0xc00019c9a0)
	/usr/local/go/src/os/exec/exec.go:567 +0x91
github.com/fluid-cloudnative/fluid/pkg/csi/fuse.(*nodeServer).NodeUnpublishVolume(0xc0004089a0, 0x167a7a0, 0xc000a1a0f0, 0xc000a280c0, 0xc0004089a0, 0x21455df, 0xc00068c0e0)
	/go/src/github.com/fluid-cloudnative/fluid/pkg/csi/fuse/nodeserver.go:141 +0xff
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler.func1(0x167a7a0, 0xc000a1a0f0, 0x13b4200, 0xc000a280c0, 0xc0008e6ae8, 0x1, 0x1, 0x30)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5612 +0x86
github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.logGRPC(0x167a7a0, 0xc000a1a0f0, 0x13b4200, 0xc000a280c0, 0xc000876100, 0xc000876120, 0xc0008e6b78, 0x4ff1e8, 0x139c040, 0xc000a1a0f0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go:99 +0x155
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler(0x138c400, 0xc0004089a0, 0x167a7a0, 0xc000a1a0f0, 0xc0008cc1e0, 0x1525910, 0x167a7a0, 0xc000a1a0f0, 0xc00039e0a0, 0x95)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5614 +0x14b
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc000450180, 0x1685b60, 0xc0006c0a80, 0xc00043ec00, 0xc000424ed0, 0x21b4d88, 0x0, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1024 +0x501
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).handleStream(0xc000450180, 0x1685b60, 0xc0006c0a80, 0xc00043ec00, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1313 +0xd3d
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc000758020, 0xc000450180, 0x1685b60, 0xc0006c0a80, 0xc00043ec00)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:722 +0xa1
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:720 +0xa1

goroutine 48 [select, 106 minutes]:
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00040e8c0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000e6c60, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000450480)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/http2_server.go:296 +0xcb
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/http2_server.go:293 +0x10c1

goroutine 49 [select, 106 minutes]:
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000450480)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/http2_server.go:968 +0x285
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/http2_server.go:302 +0x10e6

goroutine 50 [IO wait, 106 minutes]:
internal/poll.runtime_pollWait(0x7fc98a11ce08, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc000434398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000434380, 0xc000480000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000434380, 0xc000480000, 0x8000, 0x8000, 0x800010601, 0x7fc900000000, 0x106010010fcc0)
	/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0004100f8, 0xc000480000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc000400a20, 0xc00047e038, 0x9, 0x9, 0xc04e533880, 0x7fc9b0da9e98, 0x0)
	/usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x16408c0, 0xc000400a20, 0xc00047e038, 0x9, 0x9, 0x9, 0x75c025, 0xc045a4f42c, 0xc000442880)
	/usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:329
github.com/fluid-cloudnative/fluid/vendor/golang.org/x/net/http2.readFrameHeader(0xc00047e038, 0x9, 0x9, 0x16408c0, 0xc000400a20, 0x0, 0x0, 0xc045a4f420, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/golang.org/x/net/http2/frame.go:237 +0x87
github.com/fluid-cloudnative/fluid/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00047e000, 0xc045a4f420, 0x21cafc0, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000450480, 0xc000425020, 0x1525e20)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/internal/transport/http2_server.go:458 +0x98
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc000450180, 0x1685b60, 0xc000450480)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:718 +0xdb
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000450180, 0x1685b60, 0xc000450480)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:679 +0x3f
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).handleRawConn
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:678 +0x562

goroutine 886 [syscall, 8076 minutes]:
syscall.Syscall6(0xf7, 0x1, 0x2e54db, 0xc001207728, 0x1000004, 0x0, 0x0, 0x4d1d01, 0xc000a0c600, 0xc001207768)
	/usr/local/go/src/syscall/asm_linux_amd64.s:41 +0x5
os.(*Process).blockUntilWaitable(0xc000880a50, 0x203000, 0x84f8f8, 0x21cb3a0)
	/usr/local/go/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc000880a50, 0x1526c70, 0x1526c78, 0x1526c68)
	/usr/local/go/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
	/usr/local/go/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc000149ce0, 0x0, 0x0)
	/usr/local/go/src/os/exec/exec.go:507 +0x60
os/exec.(*Cmd).Run(0xc000149ce0, 0xc000c78630, 0x2136cdd)
	/usr/local/go/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc000149ce0, 0x0, 0xc0012079e8, 0x1, 0x1, 0xc000149ce0)
	/usr/local/go/src/os/exec/exec.go:567 +0x91
github.com/fluid-cloudnative/fluid/pkg/csi/fuse.(*nodeServer).NodeUnpublishVolume(0xc0004089a0, 0x167a7a0, 0xc000c78090, 0xc000fa8000, 0xc0004089a0, 0x21455df, 0xc0002cf650)
	/go/src/github.com/fluid-cloudnative/fluid/pkg/csi/fuse/nodeserver.go:141 +0xff
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler.func1(0x167a7a0, 0xc000c78090, 0x13b4200, 0xc000fa8000, 0xc000fa5ae8, 0x1, 0x1, 0x30)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5612 +0x86
github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.logGRPC(0x167a7a0, 0xc000c78090, 0x13b4200, 0xc000fa8000, 0xc000c76100, 0xc000c76120, 0xc000fa5b78, 0x4ff26d, 0x139c040, 0xc000c78090)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go:99 +0x155
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler(0x138c400, 0xc0004089a0, 0x167a7a0, 0xc000c78090, 0xc000c720c0, 0x1525910, 0x167a7a0, 0xc000c78090, 0xc000cf00a0, 0x95)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5614 +0x14b
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc000450180, 0x1685b60, 0xc0006d9680, 0xc000314500, 0xc000424ed0, 0x21b4d88, 0x0, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1024 +0x501
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).handleStream(0xc000450180, 0x1685b60, 0xc0006d9680, 0xc000314500, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1313 +0xd3d
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc000fda020, 0xc000450180, 0x1685b60, 0xc0006d9680, 0xc000314500)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:722 +0xa1
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:720 +0xa1

goroutine 4267 [IO wait, 7959 minutes]:
internal/poll.runtime_pollWait(0x7fc98a138c48, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0006991b8, 0x72, 0x201, 0x200, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0006991a0, 0xc0022c0800, 0x200, 0x200, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
os.(*File).read(...)
	/usr/local/go/src/os/file_unix.go:263
os.(*File).Read(0xc00000ea20, 0xc0022c0800, 0x200, 0x200, 0x7fc989f3e908, 0x4, 0xc001a486a0)
	/usr/local/go/src/os/file.go:116 +0x71
bytes.(*Buffer).ReadFrom(0xc00058ef00, 0x16423e0, 0xc00000ea20, 0x7fc989eac028, 0xc00058ef00, 0x45db01)
	/usr/local/go/src/bytes/buffer.go:204 +0xb1
io.copyBuffer(0x1640920, 0xc00058ef00, 0x16423e0, 0xc00000ea20, 0x0, 0x0, 0x0, 0x0, 0xc0000e2160, 0x1640b40)
	/usr/local/go/src/io/io.go:391 +0x2fc
io.Copy(...)
	/usr/local/go/src/io/io.go:364
os/exec.(*Cmd).writerDescriptor.func1(0x1526ee0, 0xc0006b96d0)
	/usr/local/go/src/os/exec/exec.go:311 +0x63
os/exec.(*Cmd).Start.func1(0xc00219db80, 0xc00072ea20)
	/usr/local/go/src/os/exec/exec.go:441 +0x27
created by os/exec.(*Cmd).Start
	/usr/local/go/src/os/exec/exec.go:440 +0x632

goroutine 3462 [syscall, 8066 minutes]:
syscall.Syscall6(0xf7, 0x1, 0x2ed471, 0xc00160f728, 0x1000004, 0x0, 0x0, 0x4d1d01, 0xc000835c80, 0xc00160f768)
	/usr/local/go/src/syscall/asm_linux_amd64.s:41 +0x5
os.(*Process).blockUntilWaitable(0xc000141c20, 0x203000, 0x84f8f8, 0x21cb3a0)
	/usr/local/go/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc000141c20, 0x1526c70, 0x1526c78, 0x1526c68)
	/usr/local/go/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
	/usr/local/go/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc000bb0c60, 0x0, 0x0)
	/usr/local/go/src/os/exec/exec.go:507 +0x60
os/exec.(*Cmd).Run(0xc000bb0c60, 0xc00082b200, 0x2136cdd)
	/usr/local/go/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc000bb0c60, 0x0, 0xc00160f9e8, 0x1, 0x1, 0xc000bb0c60)
	/usr/local/go/src/os/exec/exec.go:567 +0x91
github.com/fluid-cloudnative/fluid/pkg/csi/fuse.(*nodeServer).NodeUnpublishVolume(0xc0004089a0, 0x167a7a0, 0xc00082aa80, 0xc000440580, 0xc0004089a0, 0x21455df, 0xc0005592d0)
	/go/src/github.com/fluid-cloudnative/fluid/pkg/csi/fuse/nodeserver.go:141 +0xff
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler.func1(0x167a7a0, 0xc00082aa80, 0x13b4200, 0xc000440580, 0xc001214ae8, 0x1, 0x1, 0x30)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5612 +0x86
github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.logGRPC(0x167a7a0, 0xc00082aa80, 0x13b4200, 0xc000440580, 0xc0003c4720, 0xc0003c4740, 0xc001214b78, 0x4ff1e8, 0x139c040, 0xc00082aa80)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go:99 +0x155
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler(0x138c400, 0xc0004089a0, 0x167a7a0, 0xc00082aa80, 0xc00081ad20, 0x1525910, 0x167a7a0, 0xc00082aa80, 0xc00039e460, 0x95)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5614 +0x14b
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc000450180, 0x1685b60, 0xc00085ac00, 0xc00039c900, 0xc000424ed0, 0x21b4d88, 0x0, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1024 +0x501
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).handleStream(0xc000450180, 0x1685b60, 0xc00085ac00, 0xc00039c900, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1313 +0xd3d
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc00041c2f0, 0xc000450180, 0x1685b60, 0xc00085ac00, 0xc00039c900)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:722 +0xa1
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:720 +0xa1

goroutine 2793 [syscall, 8096 minutes]:
syscall.Syscall6(0xf7, 0x1, 0x2d80a6, 0xc000aa5728, 0x1000004, 0x0, 0x0, 0x4d1d01, 0xc00071b800, 0xc000aa5768)
	/usr/local/go/src/syscall/asm_linux_amd64.s:41 +0x5
os.(*Process).blockUntilWaitable(0xc000880a20, 0x203000, 0x84f8f8, 0x21cb3a0)
	/usr/local/go/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc000880a20, 0x1526c70, 0x1526c78, 0x1526c68)
	/usr/local/go/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
	/usr/local/go/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc000343080, 0x0, 0x0)
	/usr/local/go/src/os/exec/exec.go:507 +0x60
os/exec.(*Cmd).Run(0xc000343080, 0xc0001e5c50, 0x2136cdd)
	/usr/local/go/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc000343080, 0x0, 0xc000aa59e8, 0x1, 0x1, 0xc000343080)
	/usr/local/go/src/os/exec/exec.go:567 +0x91
github.com/fluid-cloudnative/fluid/pkg/csi/fuse.(*nodeServer).NodeUnpublishVolume(0xc0004089a0, 0x167a7a0, 0xc0001e5440, 0xc0008100c0, 0xc0004089a0, 0x21455df, 0xc000389b90)
	/go/src/github.com/fluid-cloudnative/fluid/pkg/csi/fuse/nodeserver.go:141 +0xff
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler.func1(0x167a7a0, 0xc0001e5440, 0x13b4200, 0xc0008100c0, 0xc00084aae8, 0x1, 0x1, 0x30)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5612 +0x86
github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common.logGRPC(0x167a7a0, 0xc0001e5440, 0x13b4200, 0xc0008100c0, 0xc000408180, 0xc0004081a0, 0xc00084ab78, 0x4ff1e8, 0x139c040, 0xc0001e5440)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go:99 +0x155
github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi._Node_NodeUnpublishVolume_Handler(0x138c400, 0xc0004089a0, 0x167a7a0, 0xc0001e5440, 0xc000830120, 0x1525910, 0x167a7a0, 0xc0001e5440, 0xc00039e320, 0x95)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go:5614 +0x14b
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc000450180, 0x1685b60, 0xc0006d8000, 0xc00043ea00, 0xc000424ed0, 0x21b4d88, 0x0, 0x0, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1024 +0x501
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).handleStream(0xc000450180, 0x1685b60, 0xc0006d8000, 0xc00043ea00, 0x0)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:1313 +0xd3d
github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc000b4c000, 0xc000450180, 0x1685b60, 0xc0006d8000, 0xc00043ea00)
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:722 +0xa1
created by github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc.(*Server).serveStreams.func1
	/go/src/github.com/fluid-cloudnative/fluid/vendor/google.golang.org/grpc/server.go:720 +0xa1

What you expect to happen:

How to reproduce it

Additional Information

cheyang avatar Jun 11 '21 09:06 cheyang

@cheyang hi，if @littletiger123 is busy，Can I try it please? My plan is that timeout can be a parameter passed to csi plugin and use wait.PollImmediate to try check mount/umount state until it is ready.

wangyanghack avatar Aug 04 '21 02:08 wangyanghack