server: TestSpanStatsFanOut failed

[cockroach-teamcity]

server.TestSpanStatsFanOut failed on release-24.3 @ e466862ab36572bdc894cec84536b45472bdeb5b:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x1810611]

Stack:

goroutine 2111563 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc0093c1868?, {0x757af50, 0xc0194a1d10})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x55d1080?, 0xad28d40?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc0093c1988?, {0x757af50, 0xc019663bc0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x55d1080?, 0xad28d40?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit(0x0, {0x757af50, 0xc00ec1ff50}, 0x5de)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165 +0x71
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace(0xc0093c2768, {0x757af50?, 0xc00ec1ff50?}, 0xc00f16f8c0?, 0x50?)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292 +0x6d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive(_, {_, _}, _, {_, _}, {{0xe, 0x2, 0xc0168da2a0, 0xc0168da310, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918 +0x1f9f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc009eac408, {0x757af50, 0xc019663bc0}, 0xc009051688, {0x7fa5b83513b0, 0xc00e5e5c70})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693 +0x76b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x757af50, 0xc019663bc0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00f6a03f0, {0x757af50, 0xc019663bc0}, {0x5f5fbd8, 0x1e}, 0xc004a17280)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc009eac408, {0x757af18?, 0xc005911c70?}, 0xc009051688, {0x7fa5b8351388, 0xc00e5e5c70})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206 +0xf6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc00f72a8c0, {0x75c0c58, 0xc00e5e5c70})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638 +0x32d
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x5e040e0?, 0xc00f72a8c0}, {0x75aeae8, 0xc001975aa0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267 +0xd8
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40}, 0xc0075d5a88, 0x62c0918)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160 +0x35b
google.golang.org/grpc.getChainStreamHandler.func1({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40}, 0xc0075d5a88?, 0xc00773a8c0)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174 +0x6c
google.golang.org/grpc.getChainStreamHandler.func1({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc00e7ee000?, {{0x44b305?}, {0x75c9a00?, 0xc008da3800?}}}, {0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40}, 0xc0075d5a88, 0xc0072d2a80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152 +0x384
google.golang.org/grpc.getChainStreamHandler.func1({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0xc00f6a03f0?, 0x757af50?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141 +0x27
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc00f6a03f0, {0x757af50, 0xc0194a1d10}, {0xc005207fb0, 0x29}, 0xc00677eaa0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40}, 0xc0075d5a88, 0xc0072d2a00)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140 +0xdf
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x5e040e0, 0xc00f72a8c0}, {0x75b0480, 0xc00a6efa40}, 0xc0075d5a88, 0xc013123db0?)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472 +0x85
google.golang.org/grpc.(*Server).processStreamingRPC(0xc01f15cb40, {0x75cb980, 0xc012cae680}, 0xc00137f7a0, 0xc00afa0360, 0xae02dc0, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1247
google.golang.org/grpc.(*Server).handleStream(0xc01f15cb40, {0x75cb980, 0xc012cae680}, 0xc00137f7a0, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716 +0x999
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947 +0xbb
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 2111317
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:958 +0x136

Log preceding fatal error

=== RUN   TestSpanStatsFanOut
    test_log_scope.go:165: test logs captured to: outputs.zip/logTestSpanStatsFanOut1854232148
    test_log_scope.go:76: use -show-logs to present logs inline
    test_server_shim.go:139: cluster virtualization disabled in global scope due to issue: #76378 (expected label: C-bug)

Parameters:

• attempt=1
• deadlock=true
• run=1
• shard=9

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/obs-prs

This test on roachdash | Improve this report!

Jira issue: CRDB-43413