cockroach icon indicating copy to clipboard operation
cockroach copied to clipboard

Published 20 hours ago • verified publisher icon cockroachdb

kv/kvserver: TestReplicateQueueRebalanceMultiStore failed

Open cockroach-teamcity opened this issue 1 year ago • 3 comments

kv/kvserver.TestReplicateQueueRebalanceMultiStore failed on release-24.3 @ 4cbedefd790c75cb0f21f77ed8d917c8528a7d15:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x142e911]

Stack:

goroutine 1604706 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc008a9f868?, {0x8da4b98, 0xc012914420})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689f360?, 0xd105aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc008a9f988?, {0x8da4b98, 0xc0129145a0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689f360?, 0xd105aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit(0x0, {0x8da4b98, 0xc012915110}, 0x15b6)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165 +0x71
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace(0xc008aa0768, {0x8da4b98?, 0xc012915110?}, 0xc00bdef8c0?, 0x98?)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292 +0x6d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive(_, {_, _}, _, {_, _}, {{0x62, 0x56, 0xc00ffc9570, 0xc00ffc95e0, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918 +0x1f9f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc008a1dc08, {0x8da4b98, 0xc0129145a0}, 0xc0126586c8, {0x7f36bfaf7690, 0xc0125f3580})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693 +0x76b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x8da4b98, 0xc0129145a0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc007a4ca20, {0x8da4b98, 0xc0129145a0}, {0x7478aa9, 0x1e}, 0xc006665280)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc008a1dc08, {0x8da4b60?, 0xc0127d7130?}, 0xc0126586c8, {0x7f36bfaf7668, 0xc0125f3580})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206 +0xf6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc00658d340, {0x8df2d88, 0xc0125f3580})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638 +0x32d
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x72b1ea0?, 0xc00658d340}, {0x8ddd350, 0xc0135ea8c0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267 +0xd8
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000}, 0xc011bd2510, 0x7888f78)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160 +0x35b
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000}, 0xc011bd2510?, 0xc00f8d0540)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174 +0x6c
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc008700000?, {{0x470445?}, {0x8dffa40?, 0xc01a197c20?}}}, {0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000}, 0xc011bd2510, 0xc00f8d0500)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152 +0x384
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x7f36ba0188f8?, 0x8da4b98?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141 +0x27
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc007a4ca20, {0x8da4b98, 0xc012914420}, {0xc009bc2de0, 0x29}, 0xc009d5daa0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000}, 0xc011bd2510, 0xc00f8d04c0)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140 +0xdf
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x72b1ea0, 0xc00658d340}, {0x8ddd0c8, 0xc012c1a000}, 0xc011bd2510, 0xc0125f3570?)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472 +0x85
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00d74a000, {0x8e011e0, 0xc00a036d00}, 0xc017847560, 0xc00e19b0b0, 0xd202300, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1247
google.golang.org/grpc.(*Server).handleStream(0xc00d74a000, {0x8e011e0, 0xc00a036d00}, 0xc017847560, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716 +0x999
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947 +0xbb
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1604703
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:958 +0x136
Log preceding fatal error 

*   | 	GOROOT/src/runtime/signal_unix.go:881
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13
*   | 	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140
*   | google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472
*   | google.golang.org/grpc.(*Server).processStreamingRPC
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636
*   | google.golang.org/grpc.(*Server).handleStream
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716
*   | google.golang.org/grpc.(*Server).serveStreams.func1.1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947
*   | runtime.goexit
*   | 	src/runtime/asm_amd64.s:1695
* Wraps: (2) runtime error: invalid memory address or nil pointer dereference
* Error types: (1) *withstack.withStack (2) runtime.errorString
*

Parameters:

  • attempt=1
  • run=17
  • shard=1
Help

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

  • #132905 kv/kvserver: TestReplicateQueueRebalanceMultiStore failed [A-admission-control C-test-failure O-robot P-1 T-admission-control branch-master]
/cc @cockroachdb/kv

This test on roachdash | Improve this report!

Jira issue: CRDB-43393

cockroach-teamcity avatar Oct 19 '24 12:10 cockroach-teamcity

kv/kvserver.TestReplicateQueueRebalanceMultiStore failed on release-24.3 @ e466862ab36572bdc894cec84536b45472bdeb5b:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x142e911]

Stack:

goroutine 1731942 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc00f587868?, {0x8da4b98, 0xc01c3c8cc0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689f360?, 0xd105aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc00f587988?, {0x8da4b98, 0xc01c3c8e10})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689f360?, 0xd105aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit(0x0, {0x8da4b98, 0xc01c3c9560}, 0x1210)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165 +0x71
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace(0xc00f588768, {0x8da4b98?, 0xc01c3c9560?}, 0xc0080e5080?, 0x98?)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292 +0x6d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive(_, {_, _}, _, {_, _}, {{0x11, 0x5, 0xc008fb1570, 0xc008fb15e0, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918 +0x1f9f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc003375c08, {0x8da4b98, 0xc01c3c8e10}, 0xc01331dd48, {0x7f656a0678f0, 0xc0170a0ed0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693 +0x76b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x8da4b98, 0xc01c3c8e10})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00c034ea0, {0x8da4b98, 0xc01c3c8e10}, {0x7478aa9, 0x1e}, 0xc002c6d280)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc003375c08, {0x8da4b60?, 0xc011fc8d70?}, 0xc01331dd48, {0x7f656a0678c8, 0xc0170a0ed0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206 +0xf6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc00c0308c0, {0x8df2d88, 0xc0170a0ed0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638 +0x32d
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x72b1ea0?, 0xc00c0308c0}, {0x8ddd350, 0xc014c3d180})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267 +0xd8
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0}, 0xc0134f0060, 0x7888f78)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160 +0x35b
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0}, 0xc0134f0060?, 0xc008fb8b80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174 +0x6c
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc00c062000?, {{0x470445?}, {0x8dffa40?, 0xc017081b00?}}}, {0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0}, 0xc0134f0060, 0xc008fb8b40)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152 +0x384
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x7f656c6a1ec0?, 0x8da4b98?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141 +0x27
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00c034ea0, {0x8da4b98, 0xc01c3c8cc0}, {0xc008bc7f50, 0x29}, 0xc00e541aa0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0}, 0xc0134f0060, 0xc008fb8b00)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140 +0xdf
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x72b1ea0, 0xc00c0308c0}, {0x8ddd0c8, 0xc0106824b0}, 0xc0134f0060, 0xc0170a0ec0?)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472 +0x85
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0029de3c0, {0x8e011e0, 0xc00203b380}, 0xc0108c97a0, 0xc01f9af1a0, 0xd202300, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1247
google.golang.org/grpc.(*Server).handleStream(0xc0029de3c0, {0x8e011e0, 0xc00203b380}, 0xc0108c97a0, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716 +0x999
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947 +0xbb
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1731829
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:958 +0x136
Log preceding fatal error 

*   | 	GOROOT/src/runtime/signal_unix.go:881
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13
*   | 	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140
*   | google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472
*   | google.golang.org/grpc.(*Server).processStreamingRPC
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636
*   | google.golang.org/grpc.(*Server).handleStream
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716
*   | google.golang.org/grpc.(*Server).serveStreams.func1.1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947
*   | runtime.goexit
*   | 	src/runtime/asm_amd64.s:1695
* Wraps: (2) runtime error: invalid memory address or nil pointer dereference
* Error types: (1) *withstack.withStack (2) runtime.errorString
*

Parameters:

  • attempt=1
  • run=1
  • shard=1
Help

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

  • #132905 kv/kvserver: TestReplicateQueueRebalanceMultiStore failed [A-admission-control C-test-failure O-robot P-1 T-admission-control branch-master]

This test on roachdash | Improve this report!

github-actions[bot] avatar Oct 19 '24 20:10 github-actions[bot]

kv/kvserver.TestReplicateQueueRebalanceMultiStore failed on release-24.3 @ e466862ab36572bdc894cec84536b45472bdeb5b:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x142e911]

Stack:

goroutine 1148182 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc00d547868?, {0x8da4b98, 0xc0179e10b0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689f360?, 0xd105aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc00d547988?, {0x8da4b98, 0xc0179e1230})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689f360?, 0xd105aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit(0x0, {0x8da4b98, 0xc0179e1320}, 0x56d)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165 +0x71
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace(0xc00d548768, {0x8da4b98?, 0xc0179e1320?}, 0xc008cc3760?, 0x98?)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292 +0x6d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive(_, {_, _}, _, {_, _}, {{0xe, 0x2, 0xc012a5d180, 0xc012a5d1f0, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918 +0x1f9f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc0099cd808, {0x8da4b98, 0xc0179e1230}, 0xc00c9f9448, {0x7ff389919da8, 0xc008af22b0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693 +0x76b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x8da4b98, 0xc0179e1230})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00df1a870, {0x8da4b98, 0xc0179e1230}, {0x7478aa9, 0x1e}, 0xc00605f280)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc0099cd808, {0x8da4b60?, 0xc0104cfc20?}, 0xc00c9f9448, {0x7ff389919d80, 0xc008af22b0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206 +0xf6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc00dfe7340, {0x8df2d88, 0xc008af22b0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638 +0x32d
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x72b1ea0?, 0xc00dfe7340}, {0x8ddd350, 0xc003390e40})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267 +0xd8
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0}, 0xc0097986d8, 0x7888f78)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160 +0x35b
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0}, 0xc0097986d8?, 0xc016759c00)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174 +0x6c
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc00e5ba000?, {{0x470445?}, {0x8dffa40?, 0xc01e753560?}}}, {0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0}, 0xc0097986d8, 0xc016759bc0)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152 +0x384
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x7ff385633868?, 0x8da4b98?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141 +0x27
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00df1a870, {0x8da4b98, 0xc0179e10b0}, {0xc00ebc8f30, 0x29}, 0xc0047d3aa0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0}, 0xc0097986d8, 0xc016759b80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140 +0xdf
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x72b1ea0, 0xc00dfe7340}, {0x8ddd0c8, 0xc0082042d0}, 0xc0097986d8, 0xc008af22a0?)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472 +0x85
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00b3fc3c0, {0x8e011e0, 0xc009ac2340}, 0xc004a0c5a0, 0xc00cfa7500, 0xd202300, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1247
google.golang.org/grpc.(*Server).handleStream(0xc00b3fc3c0, {0x8e011e0, 0xc009ac2340}, 0xc004a0c5a0, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716 +0x999
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947 +0xbb
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1147792
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:958 +0x136
Log preceding fatal error 

*   | 	GOROOT/src/runtime/signal_unix.go:881
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13
*   | 	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140
*   | google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472
*   | google.golang.org/grpc.(*Server).processStreamingRPC
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636
*   | google.golang.org/grpc.(*Server).handleStream
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716
*   | google.golang.org/grpc.(*Server).serveStreams.func1.1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947
*   | runtime.goexit
*   | 	src/runtime/asm_amd64.s:1695
* Wraps: (2) runtime error: invalid memory address or nil pointer dereference
* Error types: (1) *withstack.withStack (2) runtime.errorString
*

Parameters:

  • attempt=1
  • run=7
  • shard=1
Help

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

  • #132905 kv/kvserver: TestReplicateQueueRebalanceMultiStore failed [A-admission-control C-test-failure O-robot P-1 T-admission-control branch-master]

This test on roachdash | Improve this report!

cockroach-teamcity avatar Oct 20 '24 11:10 cockroach-teamcity

kv/kvserver.TestReplicateQueueRebalanceMultiStore failed with artifacts on release-24.3 @ b2d2353b876af1748607e155ecdfed9d4bba29d3:

    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [57 3 3 9 7]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [56 3 4 9 7]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [55 3 7 8 6]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [54 7 6 7 5]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [53 7 5 7 7]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [52 6 5 7 9]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [51 6 5 8 9]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [50 5 9 7 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [49 5 10 7 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [48 5 10 8 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [47 5 10 9 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [46 6 10 9 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [45 7 9 11 7]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [44 10 8 10 7]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [43 10 7 9 10]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [42 9 8 9 11]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [41 8 11 9 10]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [40 7 13 10 9]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [39 7 12 13 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [38 8 11 14 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [37 10 10 13 9]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [36 13 10 12 8]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [35 12 12 11 9]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [34 11 12 10 12]
    replicate_queue_test.go:334: current replica state (want at least 67 replicas on all stores): [74 74 75 76 74]
    replicate_queue_test.go:345: current lease state (want at least 11 leases on all stores): [33 11 12 11 12]
Help

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

  • #132905 kv/kvserver: TestReplicateQueueRebalanceMultiStore failed [A-admission-control C-test-failure O-robot P-1 T-admission-control branch-master]

This test on roachdash | Improve this report!

cockroach-teamcity avatar Oct 20 '24 22:10 cockroach-teamcity

kv/kvserver.TestReplicateQueueRebalanceMultiStore failed on release-24.3 @ b2d2353b876af1748607e155ecdfed9d4bba29d3:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x142e911]

Stack:

goroutine 1834551 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc00bfdd868?, {0x8da45f8, 0xc015373a40})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689ef80?, 0xd104aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc00bfdd988?, {0x8da45f8, 0xc015373bc0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x689ef80?, 0xd104aa0?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit(0x0, {0x8da45f8, 0xc015373ce0}, 0x562)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165 +0x71
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace(0xc00bfde768, {0x8da45f8?, 0xc015373ce0?}, 0xc00c6cab00?, 0xf8?)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292 +0x6d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive(_, {_, _}, _, {_, _}, {{0xe, 0x2, 0xc006dc1b90, 0xc006dc1c00, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918 +0x1f9f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc0095d1008, {0x8da45f8, 0xc015373bc0}, 0xc006dfb448, {0x7f6b69d6b870, 0xc01c7edde0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693 +0x76b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x8da45f8, 0xc015373bc0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00b40acf0, {0x8da45f8, 0xc015373bc0}, {0x7478553, 0x1e}, 0xc005857280)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc0095d1008, {0x8da45c0?, 0xc007d72a00?}, 0xc006dfb448, {0x7f6b69d6b848, 0xc01c7edde0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206 +0xf6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc009457180, {0x8df2768, 0xc01c7edde0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638 +0x32d
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x72b1960?, 0xc009457180}, {0x8ddcd30, 0xc0051cdec0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267 +0xd8
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00}, 0xc010224b28, 0x7888a58)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160 +0x35b
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00}, 0xc010224b28?, 0xc00e3a4bc0)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174 +0x6c
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc00c970000?, {{0x470445?}, {0x8dff420?, 0xc01c501080?}}}, {0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00}, 0xc010224b28, 0xc00e3a4b80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152 +0x384
google.golang.org/grpc.getChainStreamHandler.func1({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x7f6b6ab11140?, 0x8da45f8?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141 +0x27
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00b40acf0, {0x8da45f8, 0xc015373a40}, {0xc00e6f9920, 0x29}, 0xc007e08aa0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00}, 0xc010224b28, 0xc00e3a4b40)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140 +0xdf
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x72b1960, 0xc009457180}, {0x8ddcaa8, 0xc009c98f00}, 0xc010224b28, 0xc01c7eddd0?)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472 +0x85
google.golang.org/grpc.(*Server).processStreamingRPC(0xc015a4a960, {0x8e00bc0, 0xc00eda1380}, 0xc00628f560, 0xc019680ff0, 0xd201300, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1247
google.golang.org/grpc.(*Server).handleStream(0xc015a4a960, {0x8e00bc0, 0xc00eda1380}, 0xc00628f560, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716 +0x999
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947 +0xbb
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1834425
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:958 +0x136
Log preceding fatal error 

*   | 	GOROOT/src/runtime/signal_unix.go:881
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13
*   | 	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140
*   | google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472
*   | google.golang.org/grpc.(*Server).processStreamingRPC
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636
*   | google.golang.org/grpc.(*Server).handleStream
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716
*   | google.golang.org/grpc.(*Server).serveStreams.func1.1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947
*   | runtime.goexit
*   | 	src/runtime/asm_amd64.s:1695
* Wraps: (2) runtime error: invalid memory address or nil pointer dereference
* Error types: (1) *withstack.withStack (2) runtime.errorString
*

Parameters:

  • attempt=1
  • run=6
  • shard=1
Help

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

  • #132905 kv/kvserver: TestReplicateQueueRebalanceMultiStore failed [A-admission-control C-test-failure O-robot P-1 T-admission-control branch-master]

This test on roachdash | Improve this report!

cockroach-teamcity avatar Oct 21 '24 12:10 cockroach-teamcity

@aadityasondhi I'm assigning this to you, but I see a backport is already out for this.

arulajmani avatar Oct 21 '24 17:10 arulajmani

Merged fix in https://github.com/cockroachdb/cockroach/pull/133076.

aadityasondhi avatar Oct 21 '24 18:10 aadityasondhi