eel
eel copied to clipboard
Published
20 hours ago •
Comcast
Comcast
fatal error: concurrent map iteration and map write
I have setup a high volume pipeline with eel in the middle and found that the more workers I configure, the more frequently it crashes like this [0] when it's under high load. The attached error was caused using a worker pool of 30.
[0]
fatal error: concurrent map iteration and map write
goroutine 426404 [running]:
runtime.throw({0x974589?, 0x7f012504ca68?})
/usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc000508ea0 sp=0xc000508e70 pc=0x437531
runtime.mapiternext(0x0?)
/usr/local/go/src/runtime/map.go:871 +0x4eb fp=0xc000508f10 sp=0xc000508ea0 pc=0x41196b
runtime.mapiterinit(0x4673ae?, 0x43a0c7?, 0xc000508f48?)
/usr/local/go/src/runtime/map.go:861 +0x228 fp=0xc000508f30 sp=0xc000508f10 pc=0x411428
github.com/Comcast/eel/util.(*DefaultLogWriter).log(0xc0000aaf30, {0x95c6fd, 0x4}, {0xc0000ba570, 0x24}, 0x40f2c7?, {0xc00002efc0, 0x4, 0x1?})
/go/src/github.com/Comcast/eel/util/defaultlogger.go:94 +0x12f fp=0xc000509088 sp=0xc000508f30 pc=0x73f72f
github.com/Comcast/eel/util.(*DefaultLogger).Info(0xc000509488?, {0xc00002efc0?, 0xc0005094a8?, 0x1?})
/go/src/github.com/Comcast/eel/util/defaultlogger.go:123 +0x4e fp=0xc0005090e0 sp=0xc000509088 pc=0x73fdee
github.com/Comcast/eel/jtl.HandleEvent({0xa472e0, 0xc000132c30}, {0xa45550?, 0xc0005421c0}, 0xc00053e500)
/go/src/github.com/Comcast/eel/jtl/inproc.go:178 +0x36a8 fp=0xc0005099e0 sp=0xc0005090e0 pc=0x84b008
github.com/Comcast/eel/jtl.EventHandler({0xa45550, 0xc0005421c0}, 0x0?)
/go/src/github.com/Comcast/eel/jtl/inproc.go:32 +0x4e fp=0xc000509a20 sp=0xc0005099e0 pc=0x84792e
net/http.HandlerFunc.ServeHTTP(0x72?, {0xa45550?, 0xc0005421c0?}, 0xc00050fbd0?)
/usr/local/go/src/net/http/server.go:2084 +0x2f fp=0xc000509a48 sp=0xc000509a20 pc=0x6be78f
net/http.(*ServeMux).ServeHTTP(0x0?, {0xa45550, 0xc0005421c0}, 0xc00053e500)
/usr/local/go/src/net/http/server.go:2462 +0x149 fp=0xc000509a98 sp=0xc000509a48 pc=0x6c05e9
net/http.serverHandler.ServeHTTP({0xa43f20?}, {0xa45550, 0xc0005421c0}, 0xc00053e500)
/usr/local/go/src/net/http/server.go:2916 +0x43b fp=0xc000509b58 sp=0xc000509a98 pc=0x6c223b
net/http.(*conn).serve(0xc00023c140, {0xa45928, 0xc00030ce10})
/usr/local/go/src/net/http/server.go:1966 +0x5d7 fp=0xc000509fb8 sp=0xc000509b58 pc=0x6bd237
net/http.(*Server).Serve.func3()
/usr/local/go/src/net/http/server.go:3071 +0x2e fp=0xc000509fe0 sp=0xc000509fb8 pc=0x6c2b8e
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc000509fe8 sp=0xc000509fe0 pc=0x469581
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3071 +0x4db
goroutine 1 [chan receive, 6 minutes]:
main.main()
/go/src/github.com/Comcast/eel/main.go:210 +0x3e5
goroutine 21 [chan receive, 6 minutes]:
github.com/Comcast/eel/jtl.(*TraceLogger).processTraceLogLoop.func1()
/go/src/github.com/Comcast/eel/jtl/tracelogger.go:78 +0x18f
created by github.com/Comcast/eel/jtl.(*TraceLogger).processTraceLogLoop
/go/src/github.com/Comcast/eel/jtl/tracelogger.go:74 +0x8a
goroutine 36 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 37 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 38 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 39 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 40 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 41 [semacquire]:
sync.runtime_Semacquire(0x10?)
/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0004df6b0?)
/usr/local/go/src/sync/waitgroup.go:136 +0x52
github.com/Comcast/eel/jtl.handleEvent({0xa472e0, 0xc000590460}, 0xa40530?, 0xc0003bd3e0, {0x0?, 0x1?}, 0x0?, 0x0?)
/go/src/github.com/Comcast/eel/jtl/outproc.go:169 +0x2c5
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:86 +0x85
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 42 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 43 [runnable]:
github.com/Comcast/eel/jtl.(*JExprItem).parse(0xc000132cd0, {0xc0000c1db0?, 0x8?})
/go/src/github.com/Comcast/eel/jtl/parser.go:110 +0x96
github.com/Comcast/eel/jtl.NewJExpr({0xc0000c1db0, 0xa})
/go/src/github.com/Comcast/eel/jtl/parser.go:70 +0xe8
github.com/Comcast/eel/jtl.(*JDoc).ParseExpression(0xc0000c1db0?, {0xa472e0, 0xc000132c30}, {0x8cdd80?, 0xc0004fac80?})
/go/src/github.com/Comcast/eel/jtl/jpath.go:885 +0x65
github.com/Comcast/eel/jtl.(*JDoc).GetStringValueForExpression(0xa472e0?, {0xa472e0, 0xc000132c30}, {0xc0000c1db0?, 0xc000302800?})
/go/src/github.com/Comcast/eel/jtl/jpath.go:967 +0x55
github.com/Comcast/eel/jtl.(*HandlerFactory).GetHandlersForEvent(0xc0003022e0, {0xa472e0, 0xc000132c30}, 0x8cdd80?)
/go/src/github.com/Comcast/eel/jtl/handlers.go:399 +0xa5
github.com/Comcast/eel/jtl.handleEvent({0xa472e0, 0xc000132c30}, 0xa40530?, 0xc00031acf0, {0x0?, 0x1?}, 0x0?, 0x0?)
/go/src/github.com/Comcast/eel/jtl/outproc.go:31 +0xda
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:86 +0x85
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 44 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 45 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 46 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 47 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 48 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 49 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 50 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 51 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 52 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 53 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 54 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 55 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 56 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 57 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 58 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 59 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 60 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 61 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 62 [semacquire]:
sync.runtime_Semacquire(0x10?)
/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc000500ca0?)
/usr/local/go/src/sync/waitgroup.go:136 +0x52
github.com/Comcast/eel/jtl.handleEvent({0xa472e0, 0xc000537900}, 0xa40530?, 0xc000341020, {0x0?, 0x1?}, 0x0?, 0x0?)
/go/src/github.com/Comcast/eel/jtl/outproc.go:169 +0x2c5
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:86 +0x85
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 63 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 64 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 65 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 66 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 67 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 68 [select]:
github.com/Comcast/eel/jtl.(*Worker).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:82 +0xfd
created by github.com/Comcast/eel/jtl.(*Worker).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:74 +0x56
goroutine 69 [select]:
github.com/Comcast/eel/jtl.(*WorkDispatcher).Start.func1()
/go/src/github.com/Comcast/eel/jtl/workerpool.go:125 +0x12b
created by github.com/Comcast/eel/jtl.(*WorkDispatcher).Start
/go/src/github.com/Comcast/eel/jtl/workerpool.go:122 +0x2ae
goroutine 70 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f00fe1d8dc8, 0x72)
/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000314a00?, 0xc00003b900?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0xc000314a00)
/usr/local/go/src/internal/poll/fd_unix.go:614 +0x22c
net.(*netFD).accept(0xc000314a00)
/usr/local/go/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc00031a8e8)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x28
net.(*TCPListener).Accept(0xc00031a8e8)
/usr/local/go/src/net/tcpsock.go:288 +0x3d
net/http.(*Server).Serve(0xc0003a0000, {0xa45340, 0xc00031a8e8})
/usr/local/go/src/net/http/server.go:3039 +0x385
net/http.(*Server).ListenAndServe(0xc0003a0000)
/usr/local/go/src/net/http/server.go:2968 +0x7d
net/http.ListenAndServe(...)
/usr/local/go/src/net/http/server.go:3222
github.com/Comcast/eel/jtl.(*WebhookPlugin).startWebhookServices(0xc000302fe0, {0xa472e0?, 0xc00009ebe0})
/go/src/github.com/Comcast/eel/jtl/webhookplugin.go:75 +0x530
created by github.com/Comcast/eel/jtl.(*WebhookPlugin).StartWebhookConsumer
/go/src/github.com/Comcast/eel/jtl/webhookplugin.go:57 +0x10e
goroutine 512878 [select]:
net/http.(*persistConn).roundTrip(0xc0003cfd40, 0xc000300e00)
/usr/local/go/src/net/http/transport.go:2620 +0x974
net/http.(*Transport).roundTrip(0xc00035e000, 0xc0000d9000)
/usr/local/go/src/net/http/transport.go:594 +0x7c9
net/http.(*Transport).RoundTrip(0xc0000d9000?, 0xa42220?)
/usr/local/go/src/net/http/roundtrip.go:17 +0x19
net/http.send(0xc0000d8f00, {0xa42220, 0xc00035e000}, {0x94c840?, 0x4cad01?, 0xd07a40?})
/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc00030c540, 0xc0000d8f00, {0xc000055000?, 0xc0003cfc20?, 0xd07a40?})
/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc00030c540, 0xc0000d8f00)
/usr/local/go/src/net/http/client.go:725 +0x8f5
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:593
github.com/Comcast/eel/util.HitEndpoint({0xa472e0, 0xc000537db0}, {0xc00033a090, 0x2f}, {0xc0003d4000, 0x2c0}, {0xc0003247a0, 0x4}, 0xc0004fde30, 0x0)
/go/src/github.com/Comcast/eel/util/http.go:155 +0xb45
github.com/Comcast/eel/util.(*DefaultRetrier).Retry(0x60, {0xa472e0, 0xc000537db0}, {0xc00033a090, 0x2f}, {0xc0003d4000, 0x2c0}, {0xc0003247a0, 0x4}, 0xc0004fde30, ...)
/go/src/github.com/Comcast/eel/util/defaultretrier.go:39 +0x1a7
github.com/Comcast/eel/util.(*DefaultRetrier).RetryEndpoint(0xc00038a7b8?, {0xa472e0?, 0xc000537db0?}, {0xc00033a090?, 0x415665?}, {0xc0003d4000?, 0x4490f3?}, {0xc0003247a0?, 0xc00038a4f8?}, 0xc0004fde30, ...)
/go/src/github.com/Comcast/eel/util/defaultretrier.go:28 +0x65
github.com/Comcast/eel/jtl.(*HttpPublisher).Publish(0xc00039ab40)
/go/src/github.com/Comcast/eel/jtl/httppublisher.go:60 +0xc3
github.com/Comcast/eel/jtl.handleEvent.func1.1({0xa472e0?, 0xc000537e00}, {0xa47368, 0xc00039ab40})
/go/src/github.com/Comcast/eel/jtl/outproc.go:149 +0x102
created by github.com/Comcast/eel/jtl.handleEvent.func1
/go/src/github.com/Comcast/eel/jtl/outproc.go:146 +0x25e5
goroutine 512948 [select]:
net/http.(*persistConn).writeLoop(0xc0003cfd40)
/usr/local/go/src/net/http/transport.go:2392 +0xf5
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1751 +0x1791
goroutine 512941 [select]:
net/http.(*Transport).getConn(0xc00035e000, 0xc0005fec00, {{}, 0x0, {0xc00033a090, 0x4}, {0xc000333548, 0x12}, 0x0})
/usr/local/go/src/net/http/transport.go:1375 +0x5c6
net/http.(*Transport).roundTrip(0xc00035e000, 0xc000419700)
/usr/local/go/src/net/http/transport.go:581 +0x76f
net/http.(*Transport).RoundTrip(0xc000419700?, 0xa42220?)
/usr/local/go/src/net/http/roundtrip.go:17 +0x19
net/http.send(0xc000419600, {0xa42220, 0xc00035e000}, {0x94c840?, 0x4cad01?, 0xd07a40?})
/usr/local/go/src/net/http/client.go:252 +0x5d8
net/http.(*Client).send(0xc00030c540, 0xc000419600, {0xc000055400?, 0xc0005410e0?, 0xd07a40?})
/usr/local/go/src/net/http/client.go:176 +0x9b
net/http.(*Client).do(0xc00030c540, 0xc000419600)
/usr/local/go/src/net/http/client.go:725 +0x8f5
net/http.(*Client).Do(...)
/usr/local/go/src/net/http/client.go:593
github.com/Comcast/eel/util.HitEndpoint({0xa472e0, 0xc000590820}, {0xc00033a090, 0x2f}, {0xc00024f900, 0x4a6}, {0xc0003247a0, 0x4}, 0xc0004833e0, 0x0)
/go/src/github.com/Comcast/eel/util/http.go:155 +0xb45
github.com/Comcast/eel/util.(*DefaultRetrier).Retry(0x60, {0xa472e0, 0xc000590820}, {0xc00033a090, 0x2f}, {0xc00024f900, 0x4a6}, {0xc0003247a0, 0x4}, 0xc0004833e0, ...)
/go/src/github.com/Comcast/eel/util/defaultretrier.go:39 +0x1a7
github.com/Comcast/eel/util.(*DefaultRetrier).RetryEndpoint(0xc00011a7b8?, {0xa472e0?, 0xc000590820?}, {0xc00033a090?, 0x415665?}, {0xc00024f900?, 0x4490f3?}, {0xc0003247a0?, 0xc00011a4f8?}, 0xc0004833e0, ...)
/go/src/github.com/Comcast/eel/util/defaultretrier.go:28 +0x65
github.com/Comcast/eel/jtl.(*HttpPublisher).Publish(0xc00023cb40)
/go/src/github.com/Comcast/eel/jtl/httppublisher.go:60 +0xc3
github.com/Comcast/eel/jtl.handleEvent.func1.1({0xa472e0?, 0xc000590870}, {0xa47368, 0xc00023cb40})
/go/src/github.com/Comcast/eel/jtl/outproc.go:149 +0x102
created by github.com/Comcast/eel/jtl.handleEvent.func1
/go/src/github.com/Comcast/eel/jtl/outproc.go:146 +0x25e5
goroutine 512860 [IO wait]:
internal/poll.runtime_pollWait(0x7f00fe1d8648, 0x72)
/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000600480?, 0xc000482c11?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc000600480, {0xc000482c11, 0x1, 0x1})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000600480, {0xc000482c11?, 0xc000286118?, 0xc000116768?})
/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000562000, {0xc000482c11?, 0xc0006a6fa0?, 0xc0001de4e0?})
/usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc000482c00)
/usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
/usr/local/go/src/net/http/server.go:668 +0xca
goroutine 512947 [IO wait]:
internal/poll.runtime_pollWait(0x7f00fe1d8af8, 0x72)
/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000314480?, 0xc0003cb000?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0xc000314480, {0xc0003cb000, 0x1000, 0x1000})
/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000314480, {0xc0003cb000?, 0x1?, 0xc00033a0a5?})
/usr/local/go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000326230, {0xc0003cb000?, 0x0?, 0xc00038a510?})
/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0003cfd40, {0xc0003cb000?, 0x4070dd?, 0x60?})
/usr/local/go/src/net/http/transport.go:1929 +0x4e
bufio.(*Reader).fill(0xc00075f6e0)
/usr/local/go/src/bufio/bufio.go:106 +0x103
bufio.(*Reader).Peek(0xc00075f6e0, 0x1)
/usr/local/go/src/bufio/bufio.go:144 +0x5d
net/http.(*persistConn).readLoop(0xc0003cfd40)
/usr/local/go/src/net/http/transport.go:2093 +0x1ac
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1750 +0x173e
goroutine 512942 [IO wait]:
internal/poll.runtime_pollWait(0x7f00fe13dca0, 0x77)
/usr/local/go/src/runtime/netpoll.go:302 +0x89
internal/poll.(*pollDesc).wait(0xc000601180?, 0x95c1ae?, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32
internal/poll.(*pollDesc).waitWrite(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).WaitWrite(...)
/usr/local/go/src/internal/poll/fd_unix.go:741
net.(*netFD).connect(0xc000601180, {0xa458b8?, 0xc0000c0020}, {0xc00049d370?, 0x40ef65?}, {0xa42660?, 0xc000029140?})
/usr/local/go/src/net/fd_unix.go:141 +0x716
net.(*netFD).dial(0xc000601180, {0xa458b8, 0xc0000c0020}, {0xa462e0?, 0x0?}, {0xa462e0?, 0xc000483c50}, 0x0?)
/usr/local/go/src/net/sock_posix.go:149 +0x394
net.socket({0xa458b8, 0xc0000c0020}, {0x95c1ae, 0x3}, 0x2, 0x1, 0x0?, 0x60?, {0xa462e0, 0x0}, ...)
/usr/local/go/src/net/sock_posix.go:70 +0x2b2
net.internetSocket({0xa458b8, 0xc0000c0020}, {0x95c1ae, 0x3}, {0xa462e0, 0x0}, {0xa462e0, 0xc000483c50}, 0xc0004df760?, 0x0, ...)
/usr/local/go/src/net/ipsock_posix.go:142 +0xf8
net.(*sysDialer).doDialTCP(0xc000601100, {0xa458b8, 0xc0000c0020}, 0x0, 0xd?)
/usr/local/go/src/net/tcpsock_posix.go:65 +0xa5
net.(*sysDialer).dialTCP(0x1?, {0xa458b8?, 0xc0000c0020?}, 0x4cad96?, 0xc0005c55f0?)
/usr/local/go/src/net/tcpsock_posix.go:61 +0x59
net.(*sysDialer).dialSingle(0xc000601100, {0xa458b8, 0xc0000c0020}, {0xa43ea8?, 0xc000483c50})
/usr/local/go/src/net/dial.go:583 +0x28b
net.(*sysDialer).dialSerial(0xc000601100, {0xa458b8, 0xc0000c0020}, {0xc000511d50?, 0x1, 0x95c5e1?})
/usr/local/go/src/net/dial.go:551 +0x312
net.(*Dialer).DialContext(0xc0003221e0, {0xa458b8, 0xc0000c0020}, {0x95c1ae, 0x3}, {0xc000333548, 0x12})
/usr/local/go/src/net/dial.go:428 +0x736
net.(*Dialer).Dial(0x7f00fe0cf060?, {0x95c1ae?, 0x12504c5b8?}, {0xc000333548?, 0x118?})
/usr/local/go/src/net/dial.go:351 +0x45
net/http.(*Transport).dial(0x9137076f19a24fbe?, {0xa458f0?, 0xc000799e00?}, {0x95c1ae?, 0xc000511c60?}, {0xc000333548?, 0xc000419600?})
/usr/local/go/src/net/http/transport.go:1172 +0x5a
net/http.(*Transport).dialConn(0xc00035e000, {0xa458f0, 0xc000799e00}, {{}, 0x0, {0xc00033a090, 0x4}, {0xc000333548, 0x12}, 0x0})
/usr/local/go/src/net/http/transport.go:1607 +0x83f
net/http.(*Transport).dialConnFor(0xa47368?, 0xc0000d44d0)
/usr/local/go/src/net/http/transport.go:1449 +0xb0
created by net/http.(*Transport).queueForDial
/usr/local/go/src/net/http/transport.go:1418 +0x3d2
goroutine 512861 [chan send]:
github.com/Comcast/eel/jtl.(*lexer).emit(...)
/go/src/github.com/Comcast/eel/jtl/lexer.go:158
github.com/Comcast/eel/jtl.lexText(0xc00002f000)
/go/src/github.com/Comcast/eel/jtl/lexer.go:201 +0x30f
github.com/Comcast/eel/jtl.(*lexer).run(0xc00002f000)
/go/src/github.com/Comcast/eel/jtl/lexer.go:141 +0x2a
created by github.com/Comcast/eel/jtl.lex
/go/src/github.com/Comcast/eel/jtl/lexer.go:134 +0xf8
{"app.id":"eel","debug_action":"event_after_transformation","destination":"unknown","env.name":"default","gears.app.id":"tenant1","gears.partner.id":"","handler":"oura-to-dgraph","instance.id":"oura-2-dgraph-etl-5b7bfc9448-gjpvl","log.id":"dc9c18bf-d5a2-40e5-8f59-0c9091c851c5","log.level":"debug","log.timestamp":"2022-04-22T15:41:07.491325766Z","payload":{"set":{"context":{"block_hash":"be342e54bc765e74ed04fd51850d9fd86abf8799807657db8ecfd32841d15aa9","block_number":5058097,"certificate_idx":null,"input_idx":null,"output_address":null,"output_idx":null,"slot":16009473,"timestamp":1607575764,"tx_hash":"ff9c00eefe24cf99d19dc5e1b405f8d3b71c4d18afb95ca2ed150daf4c936a9f","tx_idx":1},"fingerprint":"16009473.tx.175673335737938583721856672132598484816","timestamp":1607575764000,"transaction":{"fee":186276,"hash":"ff9c00eefe24cf99d19dc5e1b405f8d3b71c4d18afb95ca2ed150daf4c936a9f","input_count":1,"inputs":[{"index":0,"tx_id":"a08daab5ef88fbaaf4b869ceb2e94ba17ccda4ebc5f874bc0eb67217bf7db136"}],"metadata":null,"mint":null,"mint_count":0,"network_id":null,"output_count":1,"outputs":[{"address":"addr1qy8rdzdyv0c7j2yhnraxqxjt0t0l6nwxa0xnja4v0vjkq2vxal9tylsn3za5s72ulkv9zf6nwyp2hel82lht689cxucqa8a6ey","amount":12726539187,"assets":[]}],"total_output":12726539187,"ttl":16012994,"validity_interval_start":null},"variant":"Transaction"}},"topic":"","tx.traceId":"dc9c18bf-d5a2-40e5-8f59-0c9091c851c5"}
