traceloop
traceloop copied to clipboard
kinvolk
fatal error: concurrent map iteration and map write
On image: docker.io/kinvolk/gadget:v0.3.1
{"level":"info","msg":"BPF Tracer ready","time":"2021-12-10T14:04:22Z"}
{"level":"info","msg":"Starting proc informer","time":"2021-12-10T14:04:22Z"}
{"level":"info","msg":"Starting annotation publisher","time":"2021-12-10T14:04:22Z"}
{"level":"info","msg":"Starting updater loop","time":"2021-12-10T14:04:22Z"}
fatal error: concurrent map iteration and map write
goroutine 280 [running]:
runtime.throw(0x1608852, 0x26)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/panic.go:774 +0x72 fp=0xc012a6cf60 sp=0xc012a6cf30 pc=0x431282
runtime.mapiternext(0xc012a6d078)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/map.go:858 +0x579 fp=0xc012a6cfe8 sp=0xc012a6cf60 pc=0x4126f9
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).GetPodFromContainerID(0xc0000cf540, 0xc0129f8d80, 0x40, 0x1, 0x1, 0x0)
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:148 +0xae fp=0xc012a6d0e8 sp=0xc012a6cfe8 pc=0x12e15ee
github.com/kinvolk/traceloop/pkg/straceback.(*StraceBack).updater(0xc0001165a0, 0x0, 0x0)
/home/runner/work/traceloop/traceloop/pkg/straceback/straceback.go:402 +0x158d fp=0xc012a6dfc8 sp=0xc012a6d0e8 pc=0x130ee8d
runtime.goexit()
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc012a6dfd0 sp=0xc012a6dfc8 pc=0x45e631
created by github.com/kinvolk/traceloop/pkg/straceback.NewTracer
/home/runner/work/traceloop/traceloop/pkg/straceback/straceback.go:246 +0x114b
goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f184c498e38, 0x72, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc01298da98, 0x72, 0x0, 0x0, 0x15e6a28)
/opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc01298da80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc01298da80, 0xc00064e380, 0x7f1850709d98, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/net/fd_unix.go:238 +0x42
net.(*UnixListener).accept(0xc01297d890, 0xc000029b18, 0x40fc68, 0x30)
/opt/hostedtoolcache/go/1.13.15/x64/src/net/unixsock_posix.go:162 +0x32
net.(*UnixListener).Accept(0xc01297d890, 0x1505c40, 0xc01297d980, 0x142d9c0, 0x2212180)
/opt/hostedtoolcache/go/1.13.15/x64/src/net/unixsock.go:260 +0x47
net/http.(*Server).Serve(0xc00036c0e0, 0x1821f60, 0xc01297d890, 0x0, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/net/http/server.go:2925 +0x280
main.main()
/home/runner/work/traceloop/traceloop/traceloop.go:273 +0x1389
goroutine 6 [syscall]:
os/signal.signal_recv(0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
/opt/hostedtoolcache/go/1.13.15/x64/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/opt/hostedtoolcache/go/1.13.15/x64/src/os/signal/signal_unix.go:29 +0x41
goroutine 7 [chan receive]:
k8s.io/klog/v2.(*loggingT).flushDaemon(0x22926e0)
/home/runner/go/pkg/mod/k8s.io/klog/[email protected]/klog.go:1169 +0x8b
created by k8s.io/klog/v2.init.0
/home/runner/go/pkg/mod/k8s.io/klog/[email protected]/klog.go:417 +0xdd
goroutine 8 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0x2292600)
/home/runner/go/pkg/mod/k8s.io/[email protected]/klog.go:1010 +0x8b
created by k8s.io/klog.init.0
/home/runner/go/pkg/mod/k8s.io/[email protected]/klog.go:411 +0xd6
goroutine 11 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000248060)
/home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
/home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132
goroutine 13 [select]:
k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000248180)
/home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:231 +0x405
created by k8s.io/client-go/util/workqueue.newDelayingQueue
/home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:68 +0x184
goroutine 14 [chan receive]:
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).Run(0xc0000cf540, 0x1, 0xc00004a480)
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:296 +0x2b6
created by github.com/kinvolk/traceloop/pkg/podinformer.NewPodInformer
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:128 +0x6a7
goroutine 23 [sync.Cond.Wait]:
runtime.goparkunlock(...)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0002900c8, 0x3)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0002900b8)
/opt/hostedtoolcache/go/1.13.15/x64/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0002900a0, 0xc000120f30, 0x0, 0x0, 0x0, 0x0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:493 +0xaa
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000116120)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x40
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc00025bfb0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x5e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00025bfb0, 0x17ef3c0, 0xc00024c000, 0xc0001ac001, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xa3
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00025bfb0, 0x3b9aca00, 0x0, 0xc000392c01, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0xaa
k8s.io/apimachinery/pkg/util/wait.Until(...)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc000116120, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2f6
created by github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).Run
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:284 +0x157
goroutine 323 [syscall]:
github.com/iovisor/gobpf/elf._C2func_poll(0xc00c1cfe00, 0x8, 0x1f4, 0x0, 0x0, 0x0)
_cgo_gotypes.go:352 +0x55
github.com/iovisor/gobpf/elf.perfEventPoll(0xc000044f60, 0x8, 0x8, 0xc0003380f0, 0xc00007fe40)
/home/runner/go/pkg/mod/github.com/kinvolk/[email protected]/elf/perf.go:425 +0xe6
github.com/iovisor/gobpf/elf.(*PerfMap).PollStart.func1(0xc0003380f0, 0xc00003da80, 0xc0003948c0)
/home/runner/go/pkg/mod/github.com/kinvolk/[email protected]/elf/perf.go:329 +0x792
created by github.com/iovisor/gobpf/elf.(*PerfMap).PollStart
/home/runner/go/pkg/mod/github.com/kinvolk/[email protected]/elf/perf.go:312 +0xbf
goroutine 50 [chan receive]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00004a480, 0xc000116120)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x9f
goroutine 51 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0001d0000, 0xc064f577dead9950, 0x32468e0, 0x22921a0, 0x17fab20, 0xc00095fec0, 0xc000c3bb98, 0xc00026fe00, 0xc00004a480, 0x0, ...)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:463 +0x1ab
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0001d0000, 0xc00004a480, 0x0, 0x0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:427 +0x669
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x38
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc00033f6d0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x5e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000c3bed0, 0x17ef3a0, 0xc0006a00a0, 0x1, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xa3
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0001d0000, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x1b5
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ac010, 0xc0002ae020)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x62
goroutine 16 [select]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0001d0000, 0xc00004a480, 0xc000976300, 0xc00026fe00)
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x17e
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
/home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2bf
goroutine 279 [chan send]:
github.com/kinvolk/traceloop/pkg/procinformer.(*ProcInformer).update(0xc0002af3c0, 0xc000343dd0, 0x2)
/home/runner/work/traceloop/traceloop/pkg/procinformer/procinformer.go:131 +0x6dc
github.com/kinvolk/traceloop/pkg/procinformer.NewProcInformer.func1(0xc0002af3c0)
/home/runner/work/traceloop/traceloop/pkg/procinformer/procinformer.go:50 +0x128
created by github.com/kinvolk/traceloop/pkg/procinformer.NewProcInformer
/home/runner/work/traceloop/traceloop/pkg/procinformer/procinformer.go:41 +0xd7
goroutine 38 [IO wait]:
internal/poll.runtime_pollWait(0x7f184c498f08, 0x72, 0xffffffffffffffff)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000254398, 0x72, 0xb600, 0xb63a, 0xffffffffffffffff)
/opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000254380, 0xc00017e000, 0xb63a, 0xb63a, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000254380, 0xc00017e000, 0xb63a, 0xb63a, 0x203000, 0x7f184c441ba8, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000010a90, 0xc00017e000, 0xb63a, 0xb63a, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc00091c000, 0xc00017e000, 0xb63a, 0xb63a, 0x1f, 0x17ee400, 0xc00010f920)
/opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000071758, 0x17ee260, 0xc00091c000, 0x40d3d5, 0x1459560, 0x1587420)
/opt/hostedtoolcache/go/1.13.15/x64/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000071500, 0x17ef840, 0xc000010a90, 0x5, 0xc000010a90, 0x9)
/opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000071500, 0x0, 0x0, 0x442a9f)
/opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
/opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000071500, 0xc000374000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc00027c720, 0xc0001d0118, 0x9, 0x9, 0xc00010fcc0, 0x0, 0x90d7b5)
/opt/hostedtoolcache/go/1.13.15/x64/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x17ee0e0, 0xc00027c720, 0xc0001d0118, 0x9, 0x9, 0x9, 0xc000082060, 0x0, 0x17ee400)
/opt/hostedtoolcache/go/1.13.15/x64/src/io/io.go:310 +0x87
io.ReadFull(...)
/opt/hostedtoolcache/go/1.13.15/x64/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0001d0118, 0x9, 0x9, 0x17ee0e0, 0xc00027c720, 0x0, 0x0, 0xc0002c4918, 0x101)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001d00e0, 0xc00c8e8030, 0x0, 0x0, 0x0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0xa1
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00010ffb8, 0x0, 0x0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1819 +0xbe
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00066af00)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1741 +0xa3
created by golang.org/x/net/http2.(*Transport).newClientConn
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:705 +0x68b
goroutine 67 [sync.Cond.Wait]:
runtime.goparkunlock(...)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0002510c0, 0x2)
/opt/hostedtoolcache/go/1.13.15/x64/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0002510b0)
/opt/hostedtoolcache/go/1.13.15/x64/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0002510a8, 0xc003d84001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc000251080, 0xc003d84001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
/home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2108 +0xac
encoding/json.(*Decoder).refill(0xc000251340, 0xc00027000a, 0x9)
/opt/hostedtoolcache/go/1.13.15/x64/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc000251340, 0x0, 0x0, 0x1430c40)
/opt/hostedtoolcache/go/1.13.15/x64/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc000251340, 0x144d480, 0xc0002700e0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.13.15/x64/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00098ac90, 0xc003d88000, 0x2000, 0x2600, 0x17f8320, 0x0, 0x38)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0000cfe50, 0x0, 0x17fa760, 0xc0009bf600, 0x0, 0x0, 0x0, 0xc000976478, 0x4584d0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000990ce0, 0xc00010df58, 0x8, 0x17f8320, 0xc012af8000, 0x0, 0x0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00095fec0)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x17e
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe
goroutine 68 [chan send]:
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).syncToStdout(0xc0000cf540, 0xc012a88b20, 0x18, 0x13e2200, 0xc0001503d0)
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:238 +0x633
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).processNextItem(0xc0000cf540, 0x0)
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:185 +0xf4
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).runWorker(0xc0000cf540)
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:301 +0x2b
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000688130)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x5e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000688130, 0x17ef3c0, 0xc0003ac000, 0x1, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xa3
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000688130, 0x3b9aca00, 0x0, 0x1, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0xaa
k8s.io/apimachinery/pkg/util/wait.Until(0xc000688130, 0x3b9aca00, 0xc00004a480)
/home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90 +0x4d
created by github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).Run
/home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:293 +0x22d
