pd
pd copied to clipboard
Published
20 hours ago •
tikv
tikv
TestWatcher is flaky
Flaky Test
Which jobs are failing
panic: test timed out after 5m0s
running tests:
TestWatcher (5m0s)
goroutine 368 [running]:
testing.(*M).startAlarm.func1()
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2259 +0x259
created by time.goFunc
/opt/hostedtoolcache/go/1.21.13/x64/src/time/sleep.go:176 +0x45
goroutine 1 [chan receive, 5 minutes]:
testing.(*T).Run(0xc000281040, {0x20179fb, 0xb}, 0x20a8470)
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1649 +0x871
testing.runTests.func1(0x0?)
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2054 +0x85
testing.tRunner(0xc000281040, 0xc0005dbae8)
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0x262
testing.runTests(0xc0002ddae0?, {0x2e9f840, 0x2a, 0x2a}, {0xd0?, 0x1e?, 0x2eb0040?})
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:2052 +0x8ae
testing.(*M).Run(0xc0002ddae0)
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1925 +0xcd8
main.main()
_testmain.go:163 +0x2e5
goroutine 179 [select, 5 minutes]:
go.etcd.io/etcd/server/v3/mvcc.(*watchableStore).syncVictimsLoop(0xc0002dc640)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/mvcc/watchable_store.go:261 +0x233
created by go.etcd.io/etcd/server/v3/mvcc.newWatchableStore in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/mvcc/watchable_store.go:98 +0xc25
goroutine 175 [select, 5 minutes]:
go.etcd.io/etcd/pkg/v3/schedule.(*fifo).run(0xc00065b2c0)
/home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:143 +0x42d
created by go.etcd.io/etcd/pkg/v3/schedule.NewFIFOScheduler in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:70 +0x28a
goroutine 172 [select]:
go.etcd.io/etcd/raft/v3.(*node).run(0xc00065afc0)
/home/runner/go/pkg/mod/go.etcd.io/etcd/raft/[email protected]/node.go:341 +0xc5d
created by go.etcd.io/etcd/raft/v3.StartNode in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/raft/[email protected]/node.go:230 +0x306
goroutine 170 [select]:
go.etcd.io/etcd/server/v3/mvcc/backend.(*backend).run(0xc0002c4b40)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/mvcc/backend/backend.go:433 +0x269
created by go.etcd.io/etcd/server/v3/mvcc/backend.newBackend in goroutine 169
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/mvcc/backend/backend.go:241 +0xd56
goroutine 198 [select]:
go.etcd.io/etcd/server/v3/etcdserver.(*raftNode).start.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/raft.go:174 +0x25d
created by go.etcd.io/etcd/server/v3/etcdserver.(*raftNode).start in goroutine [182](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:183)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/raft.go:169 +0xea
goroutine 174 [select]:
go.etcd.io/etcd/server/v3/lease.(*lessor).runLoop(0xc0002c4e40)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/lease/lessor.go:616 +0x1c9
created by go.etcd.io/etcd/server/v3/lease.newLessor in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/lease/lessor.go:237 +0x5a5
goroutine 162 [chan receive, 5 minutes]:
github.com/tikv/pd/pkg/encryption.TestWatcher(0xc000281380)
/home/runner/work/pd/pd/pkg/encryption/key_manager_test.go:362 +0x914
testing.tRunner(0xc000281380, 0x20a8470)
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0x262
created by testing.(*T).Run in goroutine 1
/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1648 +0x846
goroutine 171 [select, 5 minutes]:
go.etcd.io/etcd/server/v3/wal.(*filePipeline).run(0xc000471300)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/wal/file_pipeline.go:97 +0x205
created by go.etcd.io/etcd/server/v3/wal.newFilePipeline in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/wal/file_pipeline.go:55 +0x4a9
goroutine 178 [select]:
go.etcd.io/etcd/server/v3/mvcc.(*watchableStore).syncWatchersLoop(0xc0002dc640)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/mvcc/watchable_store.go:235 +0x277
created by go.etcd.io/etcd/server/v3/mvcc.newWatchableStore in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/mvcc/watchable_store.go:97 +0xbbb
goroutine 200 [select]:
go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/fileutil/purge.go:104 +0x4c5
created by go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile in goroutine [185](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:186)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/fileutil/purge.go:56 +0x894
goroutine 182 [select]:
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).run(0xc00033ea00)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:1141 +0xbe5
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).start in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:877 +0x159a
goroutine 197 [select]:
go.etcd.io/etcd/pkg/v3/schedule.(*fifo).run(0xc000541140)
/home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:143 +0x42d
created by go.etcd.io/etcd/pkg/v3/schedule.NewFIFOScheduler in goroutine 182
/home/runner/go/pkg/mod/go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:70 +0x28a
goroutine 231 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000108370, 0x1)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:418 +0x1af
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003558f0)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:551 +0x14f
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:325 +0x269
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 230
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:322 +0x260b
goroutine 185 [select, 5 minutes]:
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).purgeFile(0xc00033ea00)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:892 +0x485
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine [186](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:187) [select, 5 minutes]:
go.etcd.io/etcd/server/v3/etcdserver.monitorFileDescriptor(0xc00033ea00?, 0xc0002b8720)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/metrics.go:237 +0x3fa
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).Start.func3()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:817 +0x65
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine [187](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:188) [select]:
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).monitorVersions(0xc00033ea00)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2542 +0x145
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine [188](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:189) [select]:
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).linearizableReadLoop(0xc00033ea00)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/v3_server.go:818 +0x171
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine 199 [select]:
go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/fileutil/purge.go:104 +0x4c5
created by go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile in goroutine 185
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/fileutil/purge.go:56 +0x894
goroutine 204 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f525c358c38, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00023cca0, 0x4af001?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00023cc80)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:611 +0x425
net.(*netFD).accept(0xc00023cc80)
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_unix.go:172 +0x3e
net.(*TCPListener).accept(0xc0003083e0)
/opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock_posix.go:152 +0x3e
net.(*TCPListener).Accept(0xc0003083e0)
/opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock.go:315 +0x65
go.etcd.io/etcd/client/pkg/v3/transport.(*keepaliveListener).Accept(0xc00044e1e0)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/transport/keepalive_listener.go:52 +0x53
go.etcd.io/etcd/client/pkg/v3/transport.(*limitListener).Accept(0xc00051e630)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/transport/limit_listen.go:46 +0x6d
github.com/soheilhy/cmux.(*cMux).Serve(0xc000307c70)
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:170 +0x13c
go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve(0xc00023cc00, 0xc00033ea00, 0xc00033c880, {0x231b880, 0xc000564e40}, 0xc000598870, 0xc0005edf18?, 0x0, {0xc0001544e0, 0x2, ...})
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:257 +0x2b6b
go.etcd.io/etcd/server/v3/embed.(*Etcd).serveClients.func1(0x0?)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:782 +0x1f0
created by go.etcd.io/etcd/server/v3/embed.(*Etcd).serveClients in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:781 +0x1412
goroutine 191 [select]:
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).monitorDowngrade(0xc00033ea00)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2708 +0x155
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine 192 [select]:
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.monitorLeader.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:316 +0x173
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine 193 [select, 5 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:262
google.golang.org/grpc.(*Server).Serve(0xc00016d200, {0x232c060, 0xc0001540a0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:883 +0x611
created by go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:563 +0x66f
goroutine 194 [select, 5 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:262
net/http.(*Server).Serve(0xc00057e000, {0x232c060, 0xc000154100})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/http/server.go:3061 +0x5e8
created by go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:569 +0xa4d
goroutine 195 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f525c358d30, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00023cba0, 0x7f525c205401?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00023cb80)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:611 +0x425
net.(*netFD).accept(0xc00023cb80)
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_unix.go:172 +0x3e
net.(*TCPListener).accept(0xc000308380)
/opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock_posix.go:152 +0x3e
net.(*TCPListener).Accept(0xc000308380)
/opt/hostedtoolcache/go/1.21.13/x64/src/net/tcpsock.go:315 +0x65
go.etcd.io/etcd/client/pkg/v3/transport.(*keepaliveListener).Accept(0xc00044e130)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/transport/keepalive_listener.go:52 +0x53
go.etcd.io/etcd/client/pkg/v3/transport.(*rwTimeoutListener).Accept(0xc0003083a0)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/transport/timeout_listener.go:36 +0x43
github.com/soheilhy/cmux.(*cMux).Serve(0xc000306410)
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:170 +0x13c
go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:575 +0x167
go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers.func3(0xc000308300)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:603 +0x196
created by go.etcd.io/etcd/server/v3/embed.(*Etcd).servePeers in goroutine 162
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/etcd.go:597 +0xe3c
goroutine 201 [select]:
go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/fileutil/purge.go:104 +0x4c5
created by go.etcd.io/etcd/client/pkg/v3/fileutil.purgeFile in goroutine 185
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/pkg/[email protected]/fileutil/purge.go:56 +0x894
goroutine 205 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0005987d0, {0x232fbb8, 0xc000306d20})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 162
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 206 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000598800, {0x232fbb8, 0xc000306d70})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 162
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 207 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000598830, {0x232fbb8, 0xc000306dc0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 162
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 235 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f525c358a48, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00023dea0, 0xc000ad4000?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00023de80, {0xc000ad4000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405
net.(*netFD).Read(0xc00023de80, {0xc000ad4000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b
net.(*conn).Read(0xc000297618, {0xc000ad4000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad
bufio.(*Reader).Read(0xc0006ba900, {0xc0001abc40, 0x9, 0x9})
/opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be
io.ReadAtLeast({0x23199e0, 0xc0006ba900}, {0xc0001abc40, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc0001abc40, 0x9, 0x9}, {0x23199e0, 0xc0006ba900})
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x9b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001abc00)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:501 +0xf5
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000660240, 0x0?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:1593 +0x2e5
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 222
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:400 +0x3176
goroutine 236 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001085a0, 0x1)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:418 +0x1af
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000355960)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:551 +0x14f
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:454 +0x185
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 222
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:452 +0x3aeb
goroutine 214 [select]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00011fb00)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:1667 +0x245
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 208
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:390 +0x3094
goroutine 211 [select]:
github.com/tikv/pd/pkg/utils/etcdutil.(*healthChecker).syncer(0xc00023da00, {0x232fbb8, 0xc000306c30})
/home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:94 +0x254
created by github.com/tikv/pd/pkg/utils/etcdutil.initHealthChecker in goroutine 162
/home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:83 +0x3d4
goroutine 212 [select]:
github.com/tikv/pd/pkg/utils/etcdutil.(*healthChecker).inspector(0xc00023da00, {0x232fbb8, 0xc000306c30})
/home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:111 +0x245
created by github.com/tikv/pd/pkg/utils/etcdutil.initHealthChecker in goroutine 162
/home/runner/work/pd/pd/pkg/utils/etcdutil/health_checker.go:85 +0x487
goroutine 215 [IO wait]:
internal/poll.runtime_pollWait(0x7f525c358b40, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00023d9a0, 0xc00068c000?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00023d980, {0xc00068c000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405
net.(*netFD).Read(0xc00023d980, {0xc00068c000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b
net.(*conn).Read(0xc000297488, {0xc00068c000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad
bufio.(*Reader).Read(0xc00059b260, {0xc00034da80, 0x9, 0x9})
/opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be
io.ReadAtLeast({0x23199e0, 0xc00059b260}, {0xc00034da80, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc00034da80, 0x9, 0x9}, {0x23199e0, 0xc00059b260})
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x9b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00034da40)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:501 +0xf5
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00011fb00, 0xc00059b2c0?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:1593 +0x2e5
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 208
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:400 +0x3176
goroutine 216 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000307720, 0x1)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:418 +0x1af
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000355260)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:551 +0x14f
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:454 +0x185
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 208
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:452 +0x3aeb
goroutine 219 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000599a80, {0x232fbb8, 0xc000307db0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 204
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 220 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000599ab0, {0x232fbb8, 0xc000307e00})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 204
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 221 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000599ae0, {0x232fbb8, 0xc000307e50})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 204
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 243 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0006c1f30, {0x232fbb8, 0xc000109220})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 211
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 244 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0006c1f60, {0x232fbb8, 0xc000109270})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 211
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 224 [chan receive, 5 minutes]:
go.etcd.io/etcd/server/v3/embed.(*serveCtx).registerGateway.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:311 +0x65
created by go.etcd.io/etcd/server/v3/embed.(*serveCtx).registerGateway in goroutine 204
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:310 +0x32b
goroutine 225 [select]:
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.monitorLeader.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:316 +0x173
go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2812 +0x96
created by go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).GoAttach in goroutine 204
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/server.go:2810 +0x1cb
goroutine 226 [select, 5 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:262
net/http.(*Server).Serve(0xc00057e1e0, {0x232c060, 0xc000155c80})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/http/server.go:3061 +0x5e8
go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve.func3(0x1b48fd0?, {0x232c060, 0xc000155c80})
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:175 +0x45
created by go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve in goroutine 204
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:174 +0x13e5
goroutine 227 [select, 5 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:262
google.golang.org/grpc.(*Server).Serve(0xc00016d800, {0x232c060, 0xc000155ca0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:883 +0x611
go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve.func4(0x1b35cb0?, {0x232c060, 0xc000155ca0})
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:181 +0x45
created by go.etcd.io/etcd/server/v3/embed.(*serveCtx).serve in goroutine 204
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/embed/serve.go:180 +0x15db
goroutine 240 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000108eb0, 0x1)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:418 +0x1af
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004d0850)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:551 +0x14f
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:325 +0x269
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 239
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:322 +0x260b
goroutine 245 [select, 5 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0006c1f90, {0x232fbb8, 0xc0001092c0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:76 +0x1e9
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 211
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/grpcsync/callback_serializer.go:52 +0x20e
goroutine 232 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006c2820)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1150 +0x41b
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 230
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:344 +0x268b
goroutine 233 [IO wait]:
internal/poll.runtime_pollWait(0x7f525c358950, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000345220, 0xc0006e6000?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000345200, {0xc0006e6000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405
net.(*netFD).Read(0xc000345200, {0xc0006e6000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b
net.(*conn).Read(0xc0002975e0, {0xc0006e6000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc000355890, {0xc0006e6000, 0x8000, 0x8000})
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/buffer.go:53 +0x318
github.com/soheilhy/cmux.(*MuxConn).Read(0xc000355880, {0xc0006e6000, 0x8000, 0x8000})
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:297 +0x4b
bufio.(*Reader).Read(0xc0006ba600, {0xc0001abb60, 0x9, 0x9})
/opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be
io.ReadAtLeast({0x23199e0, 0xc0006ba600}, {0xc0001abb60, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc0001abb60, 0x9, 0x9}, {0x23199e0, 0xc0006ba600})
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x9b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001abb20)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:501 +0xf5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006c2820, {0x232fb80, 0xc0006d0750}, 0x2ee1b00?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:633 +0x145
google.golang.org/grpc.(*Server).serveStreams(0xc00016d800, {0x232fad8, 0x2ee1b00}, {0x2338300?, 0xc0006c2820}, {0x23376c8?, 0xc000355880})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1021 +0x6bc
google.golang.org/grpc.(*Server).handleRawConn.func1()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:957 +0x87
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 230
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:956 +0x2c7
goroutine 241 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006c3380)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1150 +0x41b
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 239
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:344 +0x268b
goroutine 242 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f525c358858, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0003452a0, 0xc000b24000?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000345280, {0xc000b24000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405
net.(*netFD).Read(0xc000345280, {0xc000b24000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b
net.(*conn).Read(0xc0002975f0, {0xc000b24000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0003556d0, {0xc000b24000, 0x8000, 0x8000})
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/buffer.go:53 +0x318
github.com/soheilhy/cmux.(*MuxConn).Read(0xc0003556c0, {0xc000b24000, 0x8000, 0x8000})
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:297 +0x4b
bufio.(*Reader).Read(0xc0006bbc80, {0xc0001abee0, 0x9, 0x9})
/opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be
io.ReadAtLeast({0x23199e0, 0xc0006bbc80}, {0xc0001abee0, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc0001abee0, 0x9, 0x9}, {0x23199e0, 0xc0006bbc80})
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x9b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001abea0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:501 +0xf5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006c3380, {0x232fb80, 0xc000b1e720}, 0x2ee1b00?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:633 +0x145
google.golang.org/grpc.(*Server).serveStreams(0xc00016d800, {0x232fad8, 0x2ee1b00}, {0x2338300?, 0xc0006c3380}, {0x23376c8?, 0xc0003556c0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1021 +0x6bc
google.golang.org/grpc.(*Server).handleRawConn.func1()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:957 +0x87
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 239
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:956 +0x2c7
goroutine 264 [select, 5 minutes]:
github.com/tikv/pd/pkg/encryption.(*Manager).StartBackgroundLoop(0xc0000c1f80, {0x232fbb8, 0xc000307360})
/home/runner/work/pd/pd/pkg/encryption/key_manager.go:234 +0xc4d
created by github.com/tikv/pd/pkg/encryption.TestWatcher in goroutine 162
/home/runner/work/pd/pd/pkg/encryption/key_manager_test.go:342 +0x41a
goroutine 271 [select, 5 minutes]:
go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc000c2ad00)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/[email protected]/watch.go:545 +0x4d3
created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream in goroutine 264
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/[email protected]/watch.go:292 +0x6cd
goroutine 270 [select, 5 minutes]:
context.(*cancelCtx).propagateCancel.func2()
/opt/hostedtoolcache/go/1.21.13/x64/src/context/context.go:505 +0xe5
created by context.(*cancelCtx).propagateCancel in goroutine 264
/opt/hostedtoolcache/go/1.21.13/x64/src/context/context.go:504 +0x530
goroutine 251 [select]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0006606c0)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:1667 +0x245
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 246
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:390 +0x3094
goroutine 252 [IO wait]:
internal/poll.runtime_pollWait(0x7f525c358760, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000345fa0, 0xc000b4c000?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000345f80, {0xc000b4c000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405
net.(*netFD).Read(0xc000345f80, {0xc000b4c000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b
net.(*conn).Read(0xc000297788, {0xc000b4c000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad
bufio.(*Reader).Read(0xc000b3e4e0, {0xc000175000, 0x9, 0x9})
/opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be
io.ReadAtLeast({0x23199e0, 0xc000b3e4e0}, {0xc000175000, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc000175000, 0x9, 0x9}, {0x23199e0, 0xc000b3e4e0})
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x9b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000174fc0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:501 +0xf5
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0006606c0, 0xc000b3e540?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:1593 +0x2e5
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 246
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:400 +0x3176
goroutine 253 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001097c0, 0x1)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:418 +0x1af
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004d0af0)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:551 +0x14f
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:454 +0x185
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 246
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_client.go:452 +0x3aeb
goroutine 255 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000109950, 0x1)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:418 +0x1af
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004d0b60)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:551 +0x14f
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:325 +0x269
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 254
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:322 +0x260b
goroutine 256 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b5c340)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1150 +0x41b
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 254
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:344 +0x268b
goroutine 257 [IO wait]:
internal/poll.runtime_pollWait(0x7f525c358668, 0x72)
/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0000c0120, 0xc000b5e000?, 0x0)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:84 +0xb1
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000c0100, {0xc000b5e000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/internal/poll/fd_unix.go:164 +0x405
net.(*netFD).Read(0xc0000c0100, {0xc000b5e000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/fd_posix.go:55 +0x4b
net.(*conn).Read(0xc000297778, {0xc000b5e000, 0x8000, 0x8000})
/opt/hostedtoolcache/go/1.21.13/x64/src/net/net.go:185 +0xad
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0004d0a90, {0xc000b5e000, 0x8000, 0x8000})
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/buffer.go:53 +0x318
github.com/soheilhy/cmux.(*MuxConn).Read(0xc0004d0a80, {0xc000b5e000, 0x8000, 0x8000})
/home/runner/go/pkg/mod/github.com/soheilhy/[email protected]/cmux.go:297 +0x4b
bufio.(*Reader).Read(0xc000b3e6c0, {0xc0001750e0, 0x9, 0x9})
/opt/hostedtoolcache/go/1.21.13/x64/src/bufio/bufio.go:244 +0x4be
io.ReadAtLeast({0x23199e0, 0xc000b3e6c0}, {0xc0001750e0, 0x9, 0x9}, 0x9)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc0001750e0, 0x9, 0x9}, {0x23199e0, 0xc000b3e6c0})
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x9b
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001750a0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:501 +0xf5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b5c340, {0x232fb80, 0xc000b1f8f0}, 0x2ee1b00?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:633 +0x145
google.golang.org/grpc.(*Server).serveStreams(0xc00016d800, {0x232fad8, 0x2ee1b00}, {0x2338300?, 0xc000b5c340}, {0x23376c8?, 0xc0004d0a80})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1021 +0x6bc
google.golang.org/grpc.(*Server).handleRawConn.func1()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:957 +0x87
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 254
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:956 +0x2c7
goroutine 274 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000c568c0, {0xc000c54ac0, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:193 +0x13e
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000c568c0, {0xc000c54ac0, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:173 +0x35b
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c54a80, {0xc000c54ac0, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:525 +0x5d
io.ReadAtLeast({0x231ebe0, 0xc000c54a80}, {0xc000c54ac0, 0x5, 0x5}, 0x5)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000c1fd40, {0xc000c54ac0, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:509 +0x125
google.golang.org/grpc.(*parser).recvMsg(0xc000c54ab0, 0x7fffffff)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:614 +0x6e
google.golang.org/grpc.recvAndDecompress(0xc000c54ab0, 0xc000c1fd40, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:753 +0x93
google.golang.org/grpc.recv(0x0?, {0x7f5201bebb80, 0x2ee1b00}, 0xc000ad1ecc?, {0x0, 0x0}, {0x1fc5d80, 0xc00018f810}, 0x2338300?, 0x0, ...)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:833 +0xb7
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000c2aea0, {0x1fc5d80?, 0xc00018f810}, 0x0)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:1085 +0x568
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000c1fbb4?)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:928 +0x51
google.golang.org/grpc.(*clientStream).withRetry(0xc000c1fb00, 0xc000ad1e18, 0xc000ad1e08)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:761 +0x466
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c1fb00, {0x1fc5d80?, 0xc00018f810})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:927 +0x1f6
go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc000c508d0)
/home/runner/go/pkg/mod/go.etcd.io/etcd/api/[email protected]/etcdserverpb/rpc.pb.go:6714 +0x66
go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc000c2ad00, {0x2337138, 0xc000c508d0})
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/[email protected]/watch.go:766 +0x67
created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient in goroutine 271
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/[email protected]/watch.go:921 +0x9f4
goroutine 273 [select, 5 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:392 +0xde
created by google.golang.org/grpc.newClientStreamWithParams in goroutine 271
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:391 +0x1b90
goroutine 275 [select, 5 minutes]:
go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc000c2ad00, 0xc00065fad0, 0xc000c34ea0)
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/[email protected]/watch.go:812 +0x486
created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run in goroutine 271
/home/runner/go/pkg/mod/go.etcd.io/etcd/client/[email protected]/watch.go:562 +0x8eb
goroutine 277 [select, 5 minutes]:
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch(0xc0001081e0, {0x23370e0?, 0xc000c50f80})
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:211 +0x597
go.etcd.io/etcd/api/v3/etcdserverpb._Watch_Watch_Handler({0x1def080?, 0xc0001081e0}, {0x2333b00?, 0xc000c2d818})
/home/runner/go/pkg/mod/go.etcd.io/etcd/api/[email protected]/etcdserverpb/rpc.pb.go:6743 +0xb6
github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).StreamServerInterceptor.func4({0x1def080, 0xc0001081e0}, {0x2333220?, 0xc00057ee10}, 0xc000c2d7e8, 0x20a9eb0)
/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:121 +0x15f
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func9.1({0x1def080, 0xc0001081e0}, {0x2333220, 0xc00057ee10})
/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:83 +0xc4
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.newStreamInterceptor.func1({0x1def080, 0xc0001081e0}, {0x2333220, 0xc00057ee10}, 0xc000c2d7e8, 0xc000c33540)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/interceptor.go:252 +0x6f1
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.Server.ChainStreamServer.func9({0x1def080, 0xc0001081e0}, {0x2333220, 0xc00057ee10}, 0xc000c2d7e8, 0x20a9eb0)
/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:86 +0x219
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00016d800, {0x232fb80, 0xc000c55770}, {0x2338300, 0xc0006c2820}, 0xc000c5e000, 0xc000643890, 0x2e8dd60, 0x0)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1687 +0x1f51
google.golang.org/grpc.(*Server).handleStream(0xc00016d800, {0x2338300, 0xc0006c2820}, 0xc000c5e000)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1801 +0x19f4
google.golang.org/grpc.(*Server).serveStreams.func2.1()
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1027 +0x15d
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 233
/home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1038 +0x225
goroutine 278 [select, 5 minutes]:
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*serverWatchStream).sendLoop(0xc0006d5380)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:398 +0x3cf
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch.func1()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:181 +0x2f
created by go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch in goroutine 277
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:180 +0x3e6
goroutine 279 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000c56c80, {0xc000c55810, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:181 +0x135
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000c56c80, {0xc000c55810, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:175 +0x3d5
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000c552c0, {0xc000c55810, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:525 +0x5d
io.ReadAtLeast({0x231ebe0, 0xc000c552c0}, {0xc000c55810, 0x5, 0x5}, 0x5)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:335 +0xd0
io.ReadFull(...)
/opt/hostedtoolcache/go/1.21.13/x64/src/io/io.go:354
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000c5e000, {0xc000c55810, 0x5, 0x5})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/transport.go:509 +0x125
google.golang.org/grpc.(*parser).recvMsg(0xc000c55800, 0x200000)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:614 +0x6e
google.golang.org/grpc.recvAndDecompress(0xc000c55800, 0xc000c5e000, {0x0, 0x0}, 0x200000, 0x0, {0x0, 0x0})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:753 +0x93
google.golang.org/grpc.recv(0xa25f79f1ec329ecc?, {0x7f5201be8b98, 0x2ee1b00}, 0xc000b71960?, {0x0, 0x0}, {0x1fb4be0, 0xc000c55b00}, 0xc000b71938?, 0x0, ...)
/home/runner/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:833 +0xb7
google.golang.org/grpc.(*serverStream).RecvMsg(0xc00057ee10, {0x1fb4be0?, 0xc000c55b00})
/home/runner/go/pkg/mod/google.golang.org/[email protected]/stream.go:1717 +0x311
github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredServerStream).RecvMsg(0xc000c2d818, {0x1fb4be0, 0xc000c55b00})
/home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:164 +0x53
go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchServer).Recv(0xc000c50f80)
/home/runner/go/pkg/mod/go.etcd.io/etcd/api/[email protected]/etcdserverpb/rpc.pb.go:6762 +0x66
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*serverWatchStream).recvLoop(0xc0006d5380)
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:242 +0x5e
go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch.func2()
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:191 +0x58
created by go.etcd.io/etcd/server/v3/etcdserver/api/v3rpc.(*watchServer).Watch in goroutine 277
/home/runner/go/pkg/mod/go.etcd.io/etcd/server/[email protected]/etcdserver/api/v3rpc/watch.go:[190](https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516#step:4:191) +0x4f6
make: *** [Makefile:257: ci-test-job] Error 1
Error: Process completed with exit code 2.
CI link
https://github.com/tikv/pd/actions/runs/10483693781/job/29036819007?pr=8516
