tyk
tyk copied to clipboard
TykTechnologies
[TT-5486] panic: fatal error: concurrent map iteration and map write
Branch/Environment/Version
- Branch/Version: Master/4.1.0
- Environment: hybrid
Describe the bug System running fine for days, then log outputs the below, and container eventually dies
Reproduction steps not this
Logs (debug mode or log file):
fatal error: concurrent map iteration and map write goroutine 2645490 [running]: runtime.throw(0x18e38a5, 0x26) /usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc005346130 sp=0xc005346100 pc=0x438652 runtime.mapiternext(0xc005346208) /usr/local/go/src/runtime/map.go:858 +0x54c fp=0xc0053461b0 sp=0xc005346130 pc=0x410d2c runtime.mapiterinit(0x1659e60, 0xc0055d60c0, 0xc005346208) /usr/local/go/src/runtime/map.go:848 +0x1c5 fp=0xc0053461d0 sp=0xc0053461b0 pc=0x4106e5 github.com/TykTechnologies/tyk/user.cloneAccess(0xc0055d60c0, 0x20) /app/tyk-gateway/user/session.go:203 +0xba fp=0xc005346370 sp=0xc0053461d0 pc=0xda0ffa github.com/TykTechnologies/tyk/user.SessionState.Clone(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3807d06f, 0xed9d5ae3d, 0x0, ...) /app/tyk-gateway/user/session.go:147 +0xc5 fp=0xc0053465a0 sp=0xc005346370 pc=0xda0a25 github.com/TykTechnologies/tyk/gateway.BaseMiddleware.CheckSessionAndIdentityForValidKey(0xc002d62000, 0x1b88d08, 0xc00057af00, 0xc00330b650, 0xc000024000, 0xc004f87c60, 0x20, 0xc002feb700, 0x0, 0x0, ...) /app/tyk-gateway/gateway/middleware.go:666 +0x1718 fp=0xc00534b278 sp=0xc0053465a0 pc=0x13f1d98 github.com/TykTechnologies/tyk/gateway.(*AuthKey).ProcessRequest(0xc002e8d6e0, 0x1b8a748, 0xc000774328, 0xc002feb700, 0x0, 0x0, 0x14ad369, 0xc002d62000, 0x1b88d08) /app/tyk-gateway/gateway/mw_auth_key.go:107 +0x2c5 fp=0xc00534db60 sp=0xc00534b278 pc=0x13f68c5 github.com/TykTechnologies/tyk/gateway.TraceMiddleware.ProcessRequest(0x1b9fd38, 0xc002e8d6e0, 0x1b8a748, 0xc000774328, 0xc002feb700, 0x0, 0x0, 0x0, 0x0, 0x0) /app/tyk-gateway/gateway/middleware.go:65 +0x293 fp=0xc00534dc00 sp=0xc00534db60 pc=0x13eb273 github.com/TykTechnologies/tyk/gateway.(*Gateway).createMiddleware.func1.1(0x1b8a748, 0xc000774328, 0xc002feb700) /app/tyk-gateway/gateway/middleware.go:130 +0x53f fp=0xc00534fc60 sp=0xc00534dc00 pc=0x1492f9f net/http.HandlerFunc.ServeHTTP(0xc00061f630, 0x1b8a748, 0xc000774328, 0xc002feb700) /usr/local/go/src/net/http/server.go:2049 +0x44 fp=0xc00534fc88 sp=0xc00534fc60 pc=0x735e04 github.com/TykTechnologies/tyk/gateway.(*Gateway).createMiddleware.func1.1(0x1b8a748, 0xc000774328, 0xc002feb700) /app/tyk-gateway/gateway/middleware.go:165 +0xf9c fp=0xc005351ce8 sp=0xc00534fc88 pc=0x14939fc net/http.HandlerFunc.ServeHTTP(0xc00061f680, 0x1b8a748, 0xc000774328, 0xc002feb700) /usr/local/go/src/net/http/server.go:2049 +0x44 fp=0xc005351d10 sp=0xc005351ce8 pc=0x735e04 github.com/TykTechnologies/tyk/gateway.(*Gateway).createMiddleware.func1.1(0x1b8a748, 0xc000774328, 0xc002feb700) /app/tyk-gateway/gateway/middleware.go:165 +0xf9c fp=0xc005353d70 sp=0xc005351d10 pc=0x14939fc net/http.HandlerFunc.ServeHTTP(0xc00061f6d0, 0x1b8a748, 0xc000774328, 0xc002feb700) /usr/local/go/src/net/http/server.go:2049 +0x44 fp=0xc005353d98 sp=0xc005353d70 pc=0x735e04 github.com/gorilla/mux.(*Router).ServeHTTP(0xc000af8fc0, 0x1b8a748, 0xc000774328, 0xc004b2b500) /go/pkg/mod/github.com/gorilla/[email protected]/mux.go:210 +0xd3 fp=0xc005353ed0 sp=0xc005353d98 pc=0xbc5b13 github.com/TykTechnologies/tyk/gateway.(*handleWrapper).ServeHTTP(0xc0007740b0, 0x1b8a748, 0xc000774328, 0xc004b2b500) /app/tyk-gateway/gateway/proxy_muxer.go:52 +0x15b fp=0xc005353f40 sp=0xc005353ed0 pc=0x144a8db net/http.Handler.ServeHTTP-fm(0x1b8a748, 0xc000774328, 0xc004b2b500) /usr/local/go/src/net/http/server.go:87 +0x56 fp=0xc005353f70 sp=0xc005353f40 pc=0x75e1b6 golang.org/x/net/http2.(*serverConn).runHandler(0xc0042a1380, 0xc000774328, 0xc004b2b500, 0xc00465cf18) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:2176 +0x8b fp=0xc005353fc0 sp=0xc005353f70 pc=0xee2e8b runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc005353fc8 sp=0xc005353fc0 pc=0x4727e1 created by golang.org/x/net/http2.(*serverConn).processHeaders /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:1906 +0x5a5
goroutine 1 [chan receive, 3 minutes]: github.com/TykTechnologies/again.Wait(0xc0000dd6b0, 0xc0004ed7e0, 0xc0000dd6e0, 0x1b90ab8) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/again.go:377 +0x157 github.com/TykTechnologies/tyk/gateway.Start() /app/tyk-gateway/gateway/server.go:1516 +0x7c5 main.main() /app/tyk-gateway/main.go:8 +0x25
goroutine 34 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c18c0, 0xc0002acec0) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 35 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c18e0, 0xc0002acf00) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 36 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1900, 0xc0002acf40) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 37 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1910, 0xc0002acf80) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 38 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1920, 0xc0002acfc0) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 39 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1930, 0xc0002ad000) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 40 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1940, 0xc0002ad040) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 41 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1950, 0xc0002ad080) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 42 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1960, 0xc0002ad0c0) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 43 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0002c1970, 0xc0002ad100) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 11 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc000623770, 0xc0001b1940) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 12 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc000623790, 0xc0001b1a00) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 13 [select]: github.com/pmylund/go-cache.(*janitor).Run(0xc000623c00, 0xc0001b1c40) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 14 [select]: github.com/pmylund/go-cache.(*janitor).Run(0xc000623c10, 0xc0001b1c80) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 15 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc000623c20, 0xc0001b1cc0) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 16 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc000623c30, 0xc0001b1d00) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 113 [select]: github.com/TykTechnologies/tyk/gateway.(*HostCheckerManager).CheckActivePollerLoop(0xc000025840, 0x1b90ab8, 0xc0001b1a80) /app/tyk-gateway/gateway/host_checker_manager.go:107 +0x133 created by github.com/TykTechnologies/tyk/gateway.(*HostCheckerManager).Start /app/tyk-gateway/gateway/host_checker_manager.go:85 +0x65
goroutine 114 [select]: github.com/TykTechnologies/tyk/gateway.(*Gateway).initHealthCheck.func1(0xc000024000, 0x1b90ab8, 0xc0001b1a80) /app/tyk-gateway/gateway/health_check.go:72 +0x1a6 created by github.com/TykTechnologies/tyk/gateway.(*Gateway).initHealthCheck /app/tyk-gateway/gateway/health_check.go:62 +0x88
goroutine 115 [semacquire]: sync.runtime_SemacquireMutex(0xc00030b304, 0x7f4f2fc32301, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc00030b300) /usr/local/go/src/sync/mutex.go:138 +0x105 sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 github.com/sirupsen/logrus.(*MutexWrap).Lock(...) /go/pkg/mod/github.com/sirupsen/[email protected]/logger.go:53 github.com/sirupsen/logrus.(*Entry).fireHooks(0xc004bf6000) /go/pkg/mod/github.com/sirupsen/[email protected]/entry.go:244 +0x146 github.com/sirupsen/logrus.Entry.log(0xc00030b2d0, 0xc004b95980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/pkg/mod/github.com/sirupsen/[email protected]/entry.go:224 +0x116 github.com/sirupsen/logrus.(*Entry).Log(0xc004ab97a0, 0xc000000004, 0xc005d4ff98, 0x1, 0x1) /go/pkg/mod/github.com/sirupsen/[email protected]/entry.go:268 +0xf0 github.com/sirupsen/logrus.(*Logger).Log(0xc00030b2d0, 0x4, 0xc005d4ff98, 0x1, 0x1) /go/pkg/mod/github.com/sirupsen/[email protected]/logger.go:192 +0x7e github.com/sirupsen/logrus.(*Logger).Info(...) /go/pkg/mod/github.com/sirupsen/[email protected]/logger.go:206 github.com/TykTechnologies/tyk/gateway.(*AccessLogHandler).recordAccessLogWorker(0xc000024070) /app/tyk-gateway/gateway/access_log.go:97 +0x21d created by github.com/TykTechnologies/tyk/gateway.(*AccessLogHandler).Init /app/tyk-gateway/gateway/access_log.go:48 +0x1b7
goroutine 116 [runnable]: encoding/json.(*encodeState).string(0xc0003ee100, 0x18b61c2, 0xa, 0xc003613001) /usr/local/go/src/encoding/json/encode.go:1029 +0x4cb encoding/json.mapEncoder.encode(0x1964dc8, 0xc0003ee100, 0x1682c40, 0xc004e0edb0, 0x15, 0x1680100) /usr/local/go/src/encoding/json/encode.go:811 +0x2ab encoding/json.(*encodeState).reflectValue(0xc0003ee100, 0x1682c40, 0xc004e0edb0, 0x15, 0xc0006f0100) /usr/local/go/src/encoding/json/encode.go:360 +0x82 encoding/json.(*encodeState).marshal(0xc0003ee100, 0x1682c40, 0xc004e0edb0, 0x90cd0100, 0x0, 0x0) /usr/local/go/src/encoding/json/encode.go:332 +0xf9 encoding/json.Marshal(0x1682c40, 0xc004e0edb0, 0x18ae964, 0x4, 0xc004e3a9c8, 0x0, 0x19) /usr/local/go/src/encoding/json/encode.go:161 +0x52 github.com/bshuster-repo/logrus-logstash-hook.(*LogstashFormatter).FormatWithPrefix(0xc0006f3c10, 0xc004e01f10, 0x0, 0x0, 0x52ea76, 0x1a0, 0x184, 0xa, 0xc0046e1601) /go/pkg/mod/github.com/bshuster-repo/[email protected]/logstash_formatter.go:76 +0x685 github.com/bshuster-repo/logrus-logstash-hook.(*Hook).Fire(0xc000628000, 0xc004e01f10, 0x0, 0x0) /go/pkg/mod/github.com/bshuster-repo/[email protected]/logstash.go:114 +0x250 github.com/sirupsen/logrus.LevelHooks.Fire(0xc00036e4b0, 0x4, 0xc004e01f10, 0x0, 0xc0006f3d50) /go/pkg/mod/github.com/sirupsen/[email protected]/hooks.go:28 +0x93 github.com/sirupsen/logrus.(*Entry).fireHooks(0xc004e01f10) /go/pkg/mod/github.com/sirupsen/[email protected]/entry.go:246 +0x85 github.com/sirupsen/logrus.Entry.log(0xc00030b2d0, 0xc004e0ed80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/pkg/mod/github.com/sirupsen/[email protected]/entry.go:224 +0x116 github.com/sirupsen/logrus.(*Entry).Log(0xc005e75b20, 0xc000000004, 0xc0006f3f98, 0x1, 0x1) /go/pkg/mod/github.com/sirupsen/[email protected]/entry.go:268 +0xf0 github.com/sirupsen/logrus.(*Logger).Log(0xc00030b2d0, 0x4, 0xc0006f3f98, 0x1, 0x1) /go/pkg/mod/github.com/sirupsen/[email protected]/logger.go:192 +0x7e github.com/sirupsen/logrus.(*Logger).Info(...) /go/pkg/mod/github.com/sirupsen/[email protected]/logger.go:206 github.com/TykTechnologies/tyk/gateway.(*AccessLogHandler).recordAccessLogWorker(0xc000024070) /app/tyk-gateway/gateway/access_log.go:97 +0x21d created by github.com/TykTechnologies/tyk/gateway.(*AccessLogHandler).Init /app/tyk-gateway/gateway/access_log.go:48 +0x1b7
goroutine 373 [IO wait]: internal/poll.runtime_pollWait(0x7f4f300387d8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000768098, 0x72, 0x10000, 0x10000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc000768080, 0xc0009a0000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc000768080, 0xc0009a0000, 0x10000, 0x10000, 0xc0009baf36, 0x31, 0x50ca) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc000998008, 0xc0009a0000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:183 +0x91 github.com/TykTechnologies/gorpc.(*readerCounter).Read(0xc00099e000, 0xc0009a0000, 0x10000, 0x10000, 0x19, 0x1b020e6b4b6e47ee, 0xc0004e5d18) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/conn_stats.go:118 +0x55 bufio.(*Reader).fill(0xc0009881e0) /usr/local/go/src/bufio/bufio.go:101 +0x108 bufio.(*Reader).ReadByte(0xc0009881e0, 0x199, 0x1, 0x162f780) /usr/local/go/src/bufio/bufio.go:253 +0x39 compress/flate.(*decompressor).moreBits(0xc0009b0000, 0x162f780, 0xc00003b0e0) /usr/local/go/src/compress/flate/inflate.go:696 +0x37 compress/flate.(*decompressor).nextBlock(0xc0009b0000) /usr/local/go/src/compress/flate/inflate.go:303 +0x36 compress/flate.(*decompressor).Read(0xc0009b0000, 0xc0009c0000, 0x10000, 0x10000, 0x194, 0x868dab, 0x1) /usr/local/go/src/compress/flate/inflate.go:347 +0x79 bufio.(*Reader).Read(0xc000988240, 0xc0002fe020, 0x1, 0x9, 0xc00043d080, 0xc000992270, 0x87360b) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x1b6ca80, 0xc000988240, 0xc0002fe020, 0x1, 0x9, 0x1, 0x1, 0x0, 0x1bb2608) /usr/local/go/src/io/io.go:328 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:347 encoding/gob.decodeUintReader(0x1b6ca80, 0xc000988240, 0xc0002fe020, 0x9, 0x9, 0xc000992270, 0x1719e40, 0x0, 0x0) /usr/local/go/src/encoding/gob/decode.go:120 +0x6f encoding/gob.(*Decoder).recvMessage(0xc000994280, 0x1964a88) /usr/local/go/src/encoding/gob/decoder.go:81 +0x57 encoding/gob.(*Decoder).decodeTypeSequence(0xc000994280, 0x0, 0x1587720) /usr/local/go/src/encoding/gob/decoder.go:143 +0x10d encoding/gob.(*Decoder).DecodeValue(0xc000994280, 0x1587720, 0xc000992270, 0x16, 0x0, 0x0) /usr/local/go/src/encoding/gob/decoder.go:211 +0xdc encoding/gob.(*Decoder).Decode(0xc000994280, 0x1587720, 0xc000992270, 0x49449, 0x1) /usr/local/go/src/encoding/gob/decoder.go:188 +0x173 github.com/TykTechnologies/gorpc.(*messageDecoder).Decode(...) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/encoding.go:100 github.com/TykTechnologies/gorpc.clientReader(0xc000022a50, 0x1b70f80, 0xc000998008, 0xc0009921b0, 0xc0002fe018, 0xc000988180) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:684 +0x14a created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:564 +0x294
goroutine 206 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc0001aa658, 0xc00025cb70, 0xc0001a6650, 0xc000528240, 0xc0002d47e0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 85 [IO wait]: internal/poll.runtime_pollWait(0x7f4f2fb83918, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000660198, 0x72, 0x0, 0x0, 0x18b20b2) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0xc000660180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 net.(*netFD).accept(0xc000660180, 0x614d819908732780, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:172 +0x45 net.(*TCPListener).accept(0xc000450360, 0x628b56ab, 0xc000b01e50, 0x4de226) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(*TCPListener).Accept(0xc000450360, 0xc000b01ea0, 0x18, 0xc000603800, 0x739a7b) /usr/local/go/src/net/tcpsock.go:261 +0x65 net/http.(*Server).Serve(0xc0006260e0, 0x1b8ab98, 0xc000450360, 0x0, 0x0) /usr/local/go/src/net/http/server.go:2961 +0x285 created by github.com/TykTechnologies/tyk/gateway.(*proxyMux).serve /app/tyk-gateway/gateway/proxy_muxer.go:428 +0x725
goroutine 171 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc001090000, 0xc0003e4180, 0xc00030e078, 0xc0003a4180, 0xc00038c5a0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 243 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000394008, 0xc000f80180, 0xc000240018, 0xc00034a180, 0xc000320300) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 122 [select, 3 minutes]: github.com/pmylund/go-cache.(*janitor).Run(0xc0003104c0, 0xc0003d2d00) /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1079 +0xa9 created by github.com/pmylund/go-cache.runJanitor /go/pkg/mod/github.com/pmylund/[email protected]+incompatible/cache.go:1099 +0xa9
goroutine 1628225 [select]: golang.org/x/net/http2.(*serverConn).serve(0xc004c80480) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:858 +0x579 golang.org/x/net/http2.(*Server).ServeConn(0xc000936700, 0x1b9eff8, 0xc005008810, 0xc00541aaa8) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:471 +0x756 golang.org/x/net/http2/h2c.h2cHandler.ServeHTTP(0x1b6d1e0, 0xc0007740b0, 0xc000936700, 0x1b8ada8, 0xc0041b2460, 0xc004c07100) /go/pkg/mod/golang.org/x/[email protected]/http2/h2c/h2c.go:87 +0x257 github.com/TykTechnologies/tyk/gateway.(*h2cWrapper).ServeHTTP(0xc0004503c0, 0x1b8ada8, 0xc0041b2460, 0xc004c07100) /app/tyk-gateway/gateway/proxy_muxer.go:40 +0x52 net/http.serverHandler.ServeHTTP(0xc0006260e0, 0x1b8ada8, 0xc0041b2460, 0xc004c07100) /usr/local/go/src/net/http/server.go:2867 +0xa3 net/http.(*conn).serve(0xc0002b2a00, 0x1b90b60, 0xc004f983c0) /usr/local/go/src/net/http/server.go:1932 +0x8cd created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2993 +0x39b
goroutine 204 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc0001aa650, 0xc00025c990, 0xc0001a6610, 0xc0005281e0, 0xc0002d4660) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 2262614 [IO wait]: internal/poll.runtime_pollWait(0x7f4f30047138, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc006482318, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc006482300, 0xc0039e1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc006482300, 0xc0039e1000, 0x1000, 0x1000, 0x9, 0x0, 0x0) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc00338d7b8, 0xc0039e1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:183 +0x91 net/http.(*connReader).Read(0xc003ecc810, 0xc0039e1000, 0x1000, 0x1000, 0x9, 0x0, 0x0) /usr/local/go/src/net/http/server.go:780 +0x1b9 bufio.(*Reader).Read(0xc0033b8fc0, 0xc0041b23c0, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.(*multiReader).Read(0xc00335d338, 0xc0041b23c0, 0x9, 0x9, 0x40db9b, 0x0, 0xc000399d78) /usr/local/go/src/io/multi.go:26 +0xb2 golang.org/x/net/http2/h2c.(*rwConn).Read(0xc003f306f0, 0xc0041b23c0, 0x9, 0x9, 0x3, 0xc000399da8, 0xc000399da8) /go/pkg/mod/golang.org/x/[email protected]/http2/h2c/h2c.go:293 +0x52 io.ReadAtLeast(0x7f4f2fe73cb8, 0xc003f306f0, 0xc0041b23c0, 0x9, 0x9, 0x9, 0x2, 0x10103f30b40, 0x0) /usr/local/go/src/io/io.go:328 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader(0xc0041b23c0, 0x9, 0x9, 0x7f4f2fe73cb8, 0xc003f306f0, 0x0, 0x0, 0xc0046eff00, 0xc000581500) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x89 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0041b2380, 0xc000399f38, 0x0, 0x0, 0x2) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0xa5 golang.org/x/net/http2.(*serverConn).readFrames(0xc0046b2180) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:744 +0xa9 created by golang.org/x/net/http2.(*serverConn).serve /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:850 +0x366
goroutine 167 [select]: github.com/TykTechnologies/tyk/gateway.(*HostUptimeChecker).HostCheckLoop(0xc0001c4c80, 0x1b90ab8, 0xc0001b1a80) /app/tyk-gateway/gateway/host_checker.go:116 +0x216 created by github.com/TykTechnologies/tyk/gateway.(*HostUptimeChecker).Start /app/tyk-gateway/gateway/host_checker.go:389 +0xbe
goroutine 168 [select, 3 minutes]: github.com/TykTechnologies/tyk/gateway.(*HostUptimeChecker).HostReporter(0xc0001c4c80, 0x1b90ab8, 0xc0001b1a80) /app/tyk-gateway/gateway/host_checker.go:145 +0x134 created by github.com/TykTechnologies/tyk/gateway.(*HostUptimeChecker).Start /app/tyk-gateway/gateway/host_checker.go:391 +0x148
goroutine 125 [select, 3 minutes]: github.com/TykTechnologies/tyk/gateway.cleanIdleMemConnProviders(0x1b90ab8, 0xc0001b1a80) /app/tyk-gateway/gateway/reverse_proxy.go:829 +0xe7 created by github.com/TykTechnologies/tyk/gateway.(*Gateway).initialiseSystem /app/tyk-gateway/gateway/server.go:1206 +0xcd6
goroutine 126 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc0001aa650) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 127 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc001090000) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 128 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000656010) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 177 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000390020) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 178 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000656008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 179 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000394008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 180 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc00000e008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 181 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000932008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 182 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000998008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 183 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000690018) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 184 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000b0a008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 185 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc0007ec008) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 186 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc0001aa658) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 187 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc0007ec000) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 188 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000620010) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 189 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000774000) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 190 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000390000) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 191 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc000998010) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 192 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc0011b8000) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 193 [select, 3 minutes]: github.com/TykTechnologies/gorpc.clientHandleConnection(0xc000022a50, 0x1b9f158, 0xc0001aa588) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:566 +0x345 github.com/TykTechnologies/gorpc.clientHandler(0xc000022a50) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:529 +0x1ba created by github.com/TykTechnologies/gorpc.(*Client).Start /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:155 +0x15d
goroutine 402 [IO wait]: internal/poll.runtime_pollWait(0x7f4f30038c60, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc0007e2118, 0x72, 0x10000, 0x10000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc0007e2100, 0xc001100000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc0007e2100, 0xc001100000, 0x10000, 0x10000, 0xc001370f5d, 0x31, 0x50a3) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc000620010, 0xc001100000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:183 +0x91 github.com/TykTechnologies/gorpc.(*readerCounter).Read(0xc0003ea000, 0xc001100000, 0x10000, 0x10000, 0x19, 0x6f505813b74e4a7a, 0xc0004e5d18) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/conn_stats.go:118 +0x55 bufio.(*Reader).fill(0xc0001acc00) /usr/local/go/src/bufio/bufio.go:101 +0x108 bufio.(*Reader).ReadByte(0xc0001acc00, 0x199, 0x1, 0x162f780) /usr/local/go/src/bufio/bufio.go:253 +0x39 compress/flate.(*decompressor).moreBits(0xc000632000, 0x162f780, 0xc0001b2300) /usr/local/go/src/compress/flate/inflate.go:696 +0x37 compress/flate.(*decompressor).nextBlock(0xc000632000) /usr/local/go/src/compress/flate/inflate.go:303 +0x36 compress/flate.(*decompressor).Read(0xc000632000, 0xc001376000, 0x10000, 0x10000, 0x194, 0x868dab, 0x1) /usr/local/go/src/compress/flate/inflate.go:347 +0x79 bufio.(*Reader).Read(0xc0001acc60, 0xc0004c2030, 0x1, 0x9, 0xc0001b5600, 0xc00028ee70, 0x87360b) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x1b6ca80, 0xc0001acc60, 0xc0004c2030, 0x1, 0x9, 0x1, 0x1, 0x0, 0x1bb2608) /usr/local/go/src/io/io.go:328 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:347 encoding/gob.decodeUintReader(0x1b6ca80, 0xc0001acc60, 0xc0004c2030, 0x9, 0x9, 0xc00028ee70, 0x1719e40, 0x0, 0x0) /usr/local/go/src/encoding/gob/decode.go:120 +0x6f encoding/gob.(*Decoder).recvMessage(0xc000730180, 0x1964a88) /usr/local/go/src/encoding/gob/decoder.go:81 +0x57 encoding/gob.(*Decoder).decodeTypeSequence(0xc000730180, 0x0, 0x1587720) /usr/local/go/src/encoding/gob/decoder.go:143 +0x10d encoding/gob.(*Decoder).DecodeValue(0xc000730180, 0x1587720, 0xc00028ee70, 0x16, 0x0, 0x0) /usr/local/go/src/encoding/gob/decoder.go:211 +0xdc encoding/gob.(*Decoder).Decode(0xc000730180, 0x1587720, 0xc00028ee70, 0x286f2, 0x1) /usr/local/go/src/encoding/gob/decoder.go:188 +0x173 github.com/TykTechnologies/gorpc.(*messageDecoder).Decode(...) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/encoding.go:100 github.com/TykTechnologies/gorpc.clientReader(0xc000022a50, 0x1b70f80, 0xc000620010, 0xc00028eb70, 0xc0004c2028, 0xc0001acba0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:684 +0x14a created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:564 +0x294
goroutine 259 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000b0a008, 0xc000b12180, 0xc00035e018, 0xc000342180, 0xc00084e300) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 372 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000998008, 0xc0009921b0, 0xc0002fe018, 0xc0001261e0, 0xc000988120) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 83 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000690018, 0xc000250ae0, 0xc000614020, 0xc0006380c0, 0xc00020a720) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 195 [chan receive, 3 minutes]: github.com/TykTechnologies/tyk/rpc.checkDisconnect() /app/tyk-gateway/rpc/rpc_client.go:510 +0x45 created by github.com/TykTechnologies/tyk/rpc.Connect /app/tyk-gateway/rpc/rpc_client.go:305 +0xc12
goroutine 100 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000390020, 0xc0002cab40, 0xc00068c040, 0xc00038e7e0, 0xc00059e720) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 196 [IO wait]: internal/poll.runtime_pollWait(0x7f4f30047790, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc0000e2798, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc0000e2780, 0xc003013000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc0000e2780, 0xc003013000, 0x1000, 0x1000, 0x47f8c7, 0xc0031b3ce0, 0xc005278540) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc0031b8340, 0xc003013000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:183 +0x91 bufio.(*Reader).fill(0xc0031b3ce0) /usr/local/go/src/bufio/bufio.go:101 +0x108 bufio.(*Reader).ReadSlice(0xc0031b3ce0, 0xa6b10a, 0xc005278540, 0x7f4f3005bac0, 0x8, 0x18, 0xc005213a58) /usr/local/go/src/bufio/bufio.go:360 +0x3d github.com/go-redis/redis/v8/internal/proto.(*Reader).readLine(0xc0031bb1c0, 0xc00473c558, 0x0, 0x0, 0x2, 0x0) /go/pkg/mod/github.com/go-redis/redis/[email protected]/internal/proto/reader.go:72 +0x48 github.com/go-redis/redis/v8/internal/proto.(*Reader).ReadLine(0xc0031bb1c0, 0x8e0b45aac0bab, 0xc005213ab8, 0x4ee0ec, 0xc0000e2780, 0x0) /go/pkg/mod/github.com/go-redis/redis/[email protected]/internal/proto/reader.go:58 +0x2f github.com/go-redis/redis/v8/internal/proto.(*Reader).ReadReply(0xc0031bb1c0, 0x1966200, 0x72, 0x0, 0x0, 0x27bcd60) /go/pkg/mod/github.com/go-redis/redis/[email protected]/internal/proto/reader.go:96 +0x45 github.com/go-redis/redis/v8.(*Cmd).readReply(...) /go/pkg/mod/github.com/go-redis/redis/[email protected]/command.go:318 github.com/go-redis/redis/v8.(*PubSub).ReceiveTimeout.func1(0xc0031bb1c0, 0x0, 0x0) /go/pkg/mod/github.com/go-redis/redis/[email protected]/pubsub.go:379 +0x45 github.com/go-redis/redis/v8/internal/pool.(*Conn).WithReader.func1(0x1b90af0, 0xc0001a6030, 0x1ba39f8, 0x27ebf20, 0x0, 0x1758c00) /go/pkg/mod/github.com/go-redis/redis/[email protected]/internal/pool/conn.go:73 +0xb9 github.com/go-redis/redis/v8/internal.WithSpan(0x1b90af0, 0xc0001a6030, 0x18b8a7d, 0xb, 0xc005213c60, 0x0, 0x0) /go/pkg/mod/github.com/go-redis/redis/[email protected]/internal/util.go:67 +0x269 github.com/go-redis/redis/v8/internal/pool.(*Conn).WithReader(0xc0031b7180, 0x1b90af0, 0xc0001a6030, 0x0, 0xc005213ce8, 0x0, 0xc005213cf8) /go/pkg/mod/github.com/go-redis/redis/[email protected]/internal/pool/conn.go:69 +0x95 github.com/go-redis/redis/v8.(*PubSub).ReceiveTimeout(0xc000660580, 0x1b90af0, 0xc0001a6030, 0x0, 0x1637680, 0xc002cb6460, 0x0, 0x0) /go/pkg/mod/github.com/go-redis/redis/[email protected]/pubsub.go:378 +0xcc github.com/go-redis/redis/v8.(*PubSub).Receive(...) /go/pkg/mod/github.com/go-redis/redis/[email protected]/pubsub.go:394 github.com/TykTechnologies/tyk/storage.(*RedisCluster).StartPubSubHandler(0xc005213e48, 0x18ce33d, 0x19, 0xc005213e38, 0x0, 0x0) /app/tyk-gateway/storage/redis_cluster.go:673 +0x19a github.com/TykTechnologies/tyk/gateway.(*Gateway).startPubSubLoop(0xc000024000) /app/tyk-gateway/gateway/redis_signals.go:63 +0xde created by github.com/TykTechnologies/tyk/gateway.(*Gateway).start /app/tyk-gateway/gateway/server.go:1585 +0x50f
goroutine 434 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000656010, 0xc0008a2120, 0xc00030e088, 0xc00069a120, 0xc000480180) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 535 [select]: golang.org/x/net/http2.(*serverConn).serve(0xc000772900) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:858 +0x579 golang.org/x/net/http2.(*Server).ServeConn(0xc000936700, 0x1b9eff8, 0xc00065ed20, 0xc000758aa8) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:471 +0x756 golang.org/x/net/http2/h2c.h2cHandler.ServeHTTP(0x1b6d1e0, 0xc0007740b0, 0xc000936700, 0x1b8ada8, 0xc002c1a380, 0xc002fb8400) /go/pkg/mod/golang.org/x/[email protected]/http2/h2c/h2c.go:87 +0x257 github.com/TykTechnologies/tyk/gateway.(*h2cWrapper).ServeHTTP(0xc0004503c0, 0x1b8ada8, 0xc002c1a380, 0xc002fb8400) /app/tyk-gateway/gateway/proxy_muxer.go:40 +0x52 net/http.serverHandler.ServeHTTP(0xc0006260e0, 0x1b8ada8, 0xc002c1a380, 0xc002fb8400) /usr/local/go/src/net/http/server.go:2867 +0xa3 net/http.(*conn).serve(0xc0002c4320, 0x1b90b60, 0xc0004c4cc0) /usr/local/go/src/net/http/server.go:1932 +0x8cd created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2993 +0x39b
goroutine 401 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000620010, 0xc00028eb70, 0xc0004c2028, 0xc0001860c0, 0xc0001acb40) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 376 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc00000e008, 0xc0006a00f0, 0xc0004340a8, 0xc0006720c0, 0xc00084e180) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 339 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000932008, 0xc00093a180, 0xc000424018, 0xc00079c180, 0xc0007a8300) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 103 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc0007ec008, 0xc0003cd890, 0xc00035e038, 0xc000638060, 0xc0006403c0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 197 [select]: github.com/TykTechnologies/gorpc.(*Client).CallTimeout(0xc000022a50, 0x15875a0, 0xc00003a420, 0x6fc23ac00, 0x1, 0x18b77cc, 0xb, 0xc00539e090) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:211 +0xee github.com/TykTechnologies/gorpc.(*DispatcherClient).CallTimeout(0xc0001244e0, 0x18b77cc, 0xb, 0x15e4320, 0x27eddc0, 0x6fc23ac00, 0x1b7f3b0, 0xc00539e088, 0x0, 0x0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/dispatcher.go:452 +0x105 github.com/TykTechnologies/tyk/rpc.FuncClientSingleton.func1(0xc00003a3a0, 0xc00003a3a0) /app/tyk-gateway/rpc/rpc_client.go:467 +0x185 github.com/cenkalti/backoff/v4.RetryNotifyWithTimer(0xc0007d5e90, 0x1b7f400, 0xc00003a3a0, 0x0, 0x1b88dc8, 0xc0007ec018, 0x0, 0x0) /go/pkg/mod/github.com/cenkalti/backoff/[email protected]/retry.go:52 +0x102 github.com/cenkalti/backoff/v4.RetryNotify(...) /go/pkg/mod/github.com/cenkalti/backoff/[email protected]/retry.go:31 github.com/cenkalti/backoff/v4.Retry(...) /go/pkg/mod/github.com/cenkalti/backoff/[email protected]/retry.go:25 github.com/TykTechnologies/tyk/rpc.FuncClientSingleton(0x18b77cc, 0xb, 0x15e4320, 0x27eddc0, 0x0, 0x0, 0x0, 0x0) /app/tyk-gateway/rpc/rpc_client.go:463 +0x145 github.com/TykTechnologies/tyk/gateway.(*RPCStorageHandler).CheckForReload(0xc000025800, 0x0, 0x0) /app/tyk-gateway/gateway/rpc_storage_handler.go:686 +0xdc github.com/TykTechnologies/tyk/gateway.(*Gateway).rpcReloadLoop(0xc000024000, 0x0, 0x0) /app/tyk-gateway/gateway/server.go:876 +0x48 created by github.com/TykTechnologies/tyk/gateway.(*Gateway).start /app/tyk-gateway/gateway/server.go:1597 +0x450
goroutine 198 [sleep]: time.Sleep(0x2540be400) /usr/local/go/src/runtime/time.go:193 +0xd2 github.com/TykTechnologies/tyk/gateway.(*RPCStorageHandler).StartRPCKeepaliveWatcher(0xc000025800) /app/tyk-gateway/gateway/rpc_storage_handler.go:751 +0x1cc created by github.com/TykTechnologies/tyk/gateway.(*Gateway).start /app/tyk-gateway/gateway/server.go:1598 +0x472
goroutine 275 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc0007ec000, 0xc0003920f0, 0xc000728028, 0xc000528060, 0xc00020a120) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 199 [sleep]: time.Sleep(0x2540be400) /usr/local/go/src/runtime/time.go:193 +0xd2 github.com/TykTechnologies/tyk/gateway.(*RPCStorageHandler).StartRPCLoopCheck(0xc000025800, 0x0, 0x0) /app/tyk-gateway/gateway/rpc_storage_handler.go:725 +0x205 created by github.com/TykTechnologies/tyk/gateway.(*Gateway).start /app/tyk-gateway/gateway/server.go:1599 +0x4ae
goroutine 200 [select]: github.com/TykTechnologies/tyk/gateway.(*Gateway).reloadLoop(0xc000024000, 0xc0002d4420, 0x0, 0x0, 0x0) /app/tyk-gateway/gateway/server.go:927 +0xa7 created by github.com/TykTechnologies/tyk/gateway.(*Gateway).start /app/tyk-gateway/gateway/server.go:1604 +0x2d6
goroutine 201 [select, 3 minutes]: github.com/TykTechnologies/tyk/gateway.(*Gateway).reloadQueueLoop(0xc000024000, 0x0, 0x0, 0x0) /app/tyk-gateway/gateway/server.go:962 +0xb8 created by github.com/TykTechnologies/tyk/gateway.(*Gateway).start /app/tyk-gateway/gateway/server.go:1605 +0x30c
goroutine 502 [IO wait, 3 minutes]: internal/poll.runtime_pollWait(0x7f4f2fb830f0, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000862198, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc000862180, 0xc002dd4000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc000862180, 0xc002dd4000, 0x1000, 0x1000, 0x9, 0x0, 0x0) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc002f240e0, 0xc002dd4000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:183 +0x91 net/http.(*connReader).Read(0xc000f80720, 0xc002dd4000, 0x1000, 0x1000, 0x9, 0x0, 0x0) /usr/local/go/src/net/http/server.go:780 +0x1b9 bufio.(*Reader).Read(0xc002dd08a0, 0xc00037eba0, 0x9, 0x9, 0x0, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.(*multiReader).Read(0xc002f221b0, 0xc00037eba0, 0x9, 0x9, 0x40db9b, 0x0, 0xc003b0fd78) /usr/local/go/src/io/multi.go:26 +0xb2 golang.org/x/net/http2/h2c.(*rwConn).Read(0xc000f807e0, 0xc00037eba0, 0x9, 0x9, 0x3, 0xc003b0fda8, 0xc003b0fda8) /go/pkg/mod/golang.org/x/[email protected]/http2/h2c/h2c.go:293 +0x52 io.ReadAtLeast(0x7f4f2fe73cb8, 0xc000f807e0, 0xc00037eba0, 0x9, 0x9, 0x9, 0x2, 0x10102eb0360, 0x0) /usr/local/go/src/io/io.go:328 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader(0xc00037eba0, 0x9, 0x9, 0x7f4f2fe73cb8, 0xc000f807e0, 0x0, 0x0, 0xc002f266a0, 0xc00015a180) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x89 golang.org/x/net/http2.(*Framer).ReadFrame(0xc00037eb60, 0xc003b0ff38, 0x0, 0x0, 0x2) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0xa5 golang.org/x/net/http2.(*serverConn).readFrames(0xc00064af00) /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:744 +0xa9 created by golang.org/x/net/http2.(*serverConn).serve /go/pkg/mod/golang.org/x/[email protected]/http2/server.go:850 +0x366
goroutine 202 [select]: github.com/TykTechnologies/tyk/storage.(*RedisController).ConnectToRedis(0xc0000dd6e0, 0x1b90ab8, 0xc0001b1a80, 0xc0004ed770, 0xc000347000) /app/tyk-gateway/storage/redis_controller.go:141 +0x2bd created by github.com/TykTechnologies/tyk/gateway.Start /app/tyk-gateway/gateway/server.go:1474 +0x6fa
goroutine 226 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000656008, 0xc00065e240, 0xc0004342c0, 0xc000672060, 0xc000294360) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 374 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000998010, 0xc000992390, 0xc0002fe048, 0xc000126240, 0xc0009882a0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 212 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc000390000, 0xc0002ca840, 0xc00068c018, 0xc00038e660, 0xc000408300) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 278 [select]: github.com/TykTechnologies/gorpc.clientWriter(0xc000022a50, 0x1b70fa0, 0xc0001aa588, 0xc000392420, 0xc000728198, 0xc0005282a0, 0xc00020a9c0) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:614 +0x725 created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:561 +0x1f8
goroutine 172 [IO wait]: internal/poll.runtime_pollWait(0x7f4f30038b78, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000a04098, 0x72, 0x10000, 0x10000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc000a04080, 0xc001094000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc000a04080, 0xc001094000, 0x10000, 0x10000, 0xc0010b0f2c, 0x31, 0x50d4) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc001090000, 0xc001094000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:183 +0x91 github.com/TykTechnologies/gorpc.(*readerCounter).Read(0xc000450000, 0xc001094000, 0x10000, 0x10000, 0x19, 0xa91826359c59dca5, 0xc0004e5d18) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/conn_stats.go:118 +0x55 bufio.(*Reader).fill(0xc00038c660) /usr/local/go/src/bufio/bufio.go:101 +0x108 bufio.(*Reader).ReadByte(0xc00038c660, 0x199, 0x1, 0x162f780) /usr/local/go/src/bufio/bufio.go:253 +0x39 compress/flate.(*decompressor).moreBits(0xc0010a4000, 0x162f780, 0xc00003b340) /usr/local/go/src/compress/flate/inflate.go:696 +0x37 compress/flate.(*decompressor).nextBlock(0xc0010a4000) /usr/local/go/src/compress/flate/inflate.go:303 +0x36 compress/flate.(*decompressor).Read(0xc0010a4000, 0xc0005d0000, 0x10000, 0x10000, 0x194, 0x868dab, 0x1) /usr/local/go/src/compress/flate/inflate.go:347 +0x79 bufio.(*Reader).Read(0xc000408780, 0xc00035e000, 0x1, 0x9, 0xc00043d340, 0xc0003cd110, 0x87360b) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x1b6ca80, 0xc000408780, 0xc00035e000, 0x1, 0x9, 0x1, 0x1, 0x0, 0x1bb2608) /usr/local/go/src/io/io.go:328 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:347 encoding/gob.decodeUintReader(0x1b6ca80, 0xc000408780, 0xc00035e000, 0x9, 0x9, 0xc0003cd110, 0x1719e40, 0x0, 0x0) /usr/local/go/src/encoding/gob/decode.go:120 +0x6f encoding/gob.(*Decoder).recvMessage(0xc000660080, 0x1964a88) /usr/local/go/src/encoding/gob/decoder.go:81 +0x57 encoding/gob.(*Decoder).decodeTypeSequence(0xc000660080, 0x0, 0x1587720) /usr/local/go/src/encoding/gob/decoder.go:143 +0x10d encoding/gob.(*Decoder).DecodeValue(0xc000660080, 0x1587720, 0xc0003cd110, 0x16, 0x0, 0x0) /usr/local/go/src/encoding/gob/decoder.go:211 +0xdc encoding/gob.(*Decoder).Decode(0xc000660080, 0x1587720, 0xc0003cd110, 0x32858, 0x1) /usr/local/go/src/encoding/gob/decoder.go:188 +0x173 github.com/TykTechnologies/gorpc.(*messageDecoder).Decode(...) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/encoding.go:100 github.com/TykTechnologies/gorpc.clientReader(0xc000022a50, 0x1b70f80, 0xc001090000, 0xc0003e4180, 0xc00030e078, 0xc00038c600) /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:684 +0x14a created by github.com/TykTechnologies/gorpc.clientHandleConnection /go/pkg/mod/github.com/!tyk!technologies/[email protected]/client.go:564 +0x294
Hi @liuwqiang - thanks for reporting this, and sorry about the lag in replying to you.
I have a few questions for you:
- Is this still happening for you (and can you reliably trigger it)?
- Can you provide gateway debug logs for a time period when the panic is occurring?
Thanks!
