Crash (`unknown caller pc`) during full-sync

[holiman]

This was bench06, running master on Geth/v1.13.5-unstable-58297e33/linux-amd64/go1.21.4, i.e. commit 58297e33.

INFO [12-01|14:15:18.864] Imported new chain segment               number=9,346,424  hash=f24505..81ce39 blocks=160  txs=10653   mgas=1209.805 elapsed=8.035s        mgasps=150.549 age=3y10mo3w  snapdiffs=2.43MiB    triedirty=1018.01MiB
runtime: g 293973197: unexpected return pc for bytes.Compare called from 0xc11265bb80
stack: frame={sp:0xc12eb567e0, fp:0xc12eb567e8} stack=[0xc12eb3c000,0xc12eb5c000)
0x000000c12eb566e0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb566f0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56700:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56710:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56720:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56730:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56740:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56750:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56760:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56770:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56780:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56790:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567a0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567b0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567c0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567d0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567e0: <0x000000c11265bb80 >0x0000000000000001 
0x000000c12eb567f0:  0x0000000000000041  0x0000000000000011 
0x000000c12eb56800:  0x00007fb0048b7d67  0x000000c11265bba0 
0x000000c12eb56810:  0x0000000000000004  0x0000000000000024 
0x000000c12eb56820:  0x000000c09939a8d0  0x0000000000b2b405 <github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper+0x0000000000000365> 
0x000000c12eb56830:  0x000000c03e797118  0x000000c11265bb80 
0x000000c12eb56840:  0x0000000000000041  0x0000000000000041 
0x000000c12eb56850:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56860:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56870:  0x0000000001c68358  0x0000000000000000 
0x000000c12eb56880:  0x00007fb0048b75e0  0x000000c09939a940 
0x000000c12eb56890:  0x0000000000b2dd8f <github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex+0x00000000000002af>  0x000000c03e797118 
0x000000c12eb568a0:  0x000000c0f8fe7500  0x00007fb0048b75e0 
0x000000c12eb568b0:  0x0000000000000000  0x0000000000514c5d <bytes.Compare+0x000000000000001d> 
0x000000c12eb568c0:  0x000000000000002c  0x0000000000000000 
0x000000c12eb568d0:  0x000000c09939a940  0x0000000000b2bbdd <github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE+0x000000000000033d> 
0x000000c12eb568e0:  0x000000c03e797080 
fatal error: unknown caller pc
runtime stack:
runtime.throw({0x18e75c3?, 0x0?})
	runtime/panic.go:1077 +0x5c fp=0x7faff3669520 sp=0x7faff36694f0 pc=0x43c8dc
runtime.(*unwinder).next(0x7faff3669720)
	runtime/traceback.go:475 +0x249 fp=0x7faff3669598 sp=0x7faff3669520 pc=0x463329
runtime.copystack(0xc000a7bba0, 0x800000002?)
	runtime/stack.go:934 +0x2d2 fp=0x7faff3669890 sp=0x7faff3669598 pc=0x456a92
runtime.newstack()
	runtime/stack.go:1116 +0x47f fp=0x7faff3669a40 sp=0x7faff3669890 pc=0x45703f
traceback: unexpected SPWRITE function runtime.morestack
runtime.morestack()
	runtime/asm_amd64.s:593 +0x8f fp=0x7faff3669a48 sp=0x7faff3669a40 pc=0x47100f
goroutine 293973197 [copystack]:
bytes.Compare({0x1?, 0x41?, 0x11?}, {0x7fb0048b7d67?, 0xc11265bba0?, 0x4?})
	bytes/bytes.go:26 +0x46 fp=0xc12eb567e8 sp=0xc12eb567e0 pc=0x514c86
runtime: g 293973197: unexpected return pc for bytes.Compare called from 0xc11265bb80
stack: frame={sp:0xc12eb567e0, fp:0xc12eb567e8} stack=[0xc12eb3c000,0xc12eb5c000)
0x000000c12eb566e0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb566f0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56700:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56710:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56720:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56730:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56740:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56750:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56760:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56770:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56780:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56790:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567a0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567b0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567c0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567d0:  0x0000000000000000  0x0000000000000000 
0x000000c12eb567e0: <0x000000c11265bb80 >0x0000000000000001 
0x000000c12eb567f0:  0x0000000000000041  0x0000000000000011 
0x000000c12eb56800:  0x00007fb0048b7d67  0x000000c11265bba0 
0x000000c12eb56810:  0x0000000000000004  0x0000000000000024 
0x000000c12eb56820:  0x000000c09939a8d0  0x0000000000b2b405 <github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekGEHelper+0x0000000000000365> 
0x000000c12eb56830:  0x000000c03e797118  0x000000c11265bb80 
0x000000c12eb56840:  0x0000000000000041  0x0000000000000041 
0x000000c12eb56850:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56860:  0x0000000000000000  0x0000000000000000 
0x000000c12eb56870:  0x0000000001c68358  0x0000000000000000 
0x000000c12eb56880:  0x00007fb0048b75e0  0x000000c09939a940 
0x000000c12eb56890:  0x0000000000b2dd8f <github.com/cockroachdb/pebble/sstable.(*twoLevelIterator).loadIndex+0x00000000000002af>  0x000000c03e797118 
0x000000c12eb568a0:  0x000000c0f8fe7500  0x00007fb0048b75e0 
0x000000c12eb568b0:  0x0000000000000000  0x0000000000514c5d <bytes.Compare+0x000000000000001d> 
0x000000c12eb568c0:  0x000000000000002c  0x0000000000000000 
0x000000c12eb568d0:  0x000000c09939a940  0x0000000000b2bbdd <github.com/cockroachdb/pebble/sstable.(*singleLevelIterator).seekPrefixGE+0x000000000000033d> 
0x000000c12eb568e0:  0x000000c03e797080 
created by github.com/ethereum/go-ethereum/eth/downloader.(*Downloader).spawnSync in goroutine 293973192
	github.com/ethereum/go-ethereum/eth/downloader/downloader.go:654 +0x7b
goroutine 1 [chan receive, 4626 minutes]:
runtime.gopark(0xc000620f40?, 0x18d675f?, 0x4?, 0x0?, 0x18992a0?)
	runtime/proc.go:398 +0xce fp=0xc000737ac0 sp=0xc000737aa0 pc=0x43f7ce
runtime.chanrecv(0xc000156420, 0x0, 0x1)
	runtime/chan.go:583 +0x3cd fp=0xc000737b38 sp=0xc000737ac0 pc=0x40ad0d
runtime.chanrecv1(0xc000787bc8?, 0xc000228b60?)
	runtime/chan.go:442 +0x12 fp=0xc000737b60 sp=0xc000737b38 pc=0x40a912
github.com/ethereum/go-ethereum/node.(*Node).Wait(...)
	github.com/ethereum/go-ethereum/node/node.go:555
main.geth(0xc00061a000)
	github.com/ethereum/go-ethereum/cmd/geth/main.go:341 +0x13c fp=0xc000737bf8 sp=0xc000737b60 pc=0x14281bc
github.com/urfave/cli/v2.(*Command).Run(0xc00022a160, 0xc00061a000, {0xc0001c0000, 0x17, 0x18})
	github.com/urfave/cli/[email protected]/command.go:274 +0x998 fp=0xc000737e88 sp=0xc000737bf8 pc=0xf086f8
github.com/urfave/cli/v2.(*App).RunContext(0xc0006f65a0, {0x1e40e58?, 0x2b70820}, {0xc0001c0000, 0x17, 0x18})
	github.com/urfave/cli/[email protected]/app.go:332 +0x5b7 fp=0xc000737ef0 sp=0xc000737e88 pc=0xf04ef7
github.com/urfave/cli/v2.(*App).Run(...)
	github.com/urfave/cli/[email protected]/app.go:309
main.main()
	github.com/ethereum/go-ethereum/cmd/geth/main.go:267 +0x45 fp=0xc000737f40 sp=0xc000737ef0 pc=0x1427c65
runtime.main()
	runtime/proc.go:267 +0x2bb fp=0xc000737fe0 sp=0xc000737f40 pc=0x43f35b
runtime.goexit()
	runtime/asm_amd64.s:1650 +0x1 fp=0xc000737fe8 sp=0xc000737fe0 pc=0x472ea1
goroutine 2 [force gc (idle), 4626 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	runtime/proc.go:398 +0xce fp=0xc00007afa8 sp=0xc00007af88 pc=0x43f7ce
runtime.goparkunlock(...)
	runtime/proc.go:404
runtime.forcegchelper()
	runtime/proc.go:322 +0xb3 fp=0xc00007afe0 sp=0xc00007afa8 pc=0x43f633
runtime.goexit()
	runtime/asm_amd64.s:1650 +0x1 fp=0xc00007afe8 sp=0xc00007afe0 pc=0x472ea1
created by runtime.init.6 in goroutine 1
	runtime/proc.go:310 +0x1a
goroutine 18 [GC sweep wait]:
runtime.gopark(0x2b37b01?, 0x0?, 0x0?, 0x0?, 0x0?)
	runtime/proc.go:398 +0xce fp=0xc000076778 sp=0xc000076758 pc=0x43f7ce
runtime.goparkunlock(...)
	runtime/proc.go:404
runtime.bgsweep(0x0?)
	runtime/mgcsweep.go:321 +0xdf fp=0xc0000767c8 sp=0xc000076778 pc=0x4295ff
runtime.gcenable.func1()
	runtime/mgc.go:200 +0x25 fp=0xc0000767e0 sp=0xc0000767c8 pc=0x41e6e5
runtime.goexit()
	runtime/asm_amd64.s:1650 +0x1 fp=0xc0000767e8 sp=0xc0000767e0 pc=0x472ea1
created by runtime.gcenable in goroutine 1
	runtime/mgc.go:200 +0x66

Fuller dump: stackdump.txt

[holiman]

Possibly related: https://github.com/cockroachdb/pebble/issues/2507

[MariusVanDerWijden]

I tried to dig into it a bit. Looks like something smashed our stack, the only real possibility (other than the runtime) that I see when stepping through the code is the compression. We're using the default compression algorithm snappy which is implemented in ASM, maybe theres an issue there?