http-add-on fatal error: concurrent map writes

fatal error: concurrent map writes

Open abhishektvz opened this issue 3 years ago • 1 comments

Report

`{"level":"info","ts":1662539420.6792893,"logger":"startHealthcheckServer","caller":"scaler/main.go:216","msg":"starting health check server","addr":"0.0.0.0:9091"} {"level":"info","ts":1662539420.6795864,"caller":"scaler/main.go:144","msg":"starting grpc server","address":"0.0.0.0:9090"} fatal error: concurrent map writes

goroutine 151 [running]: runtime.throw({0x180ee20, 0x7000103}) /usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc0004d99c8 sp=0xc0004d9998 pc=0x433831 runtime.mapassign_faststr(0x15de900, 0xc00008f320, {0xc0004a5820, 0x1a}) /usr/local/go/src/runtime/map_faststr.go:294 +0x38b fp=0xc0004d9a30 sp=0xc0004d99c8 pc=0x412a8b main.mergeCountsWithRoutingTable(0x15df500, {0x1a068a8, 0xc00049ebe0}) /go/src/github.com/kedacore/http-add-on/scaler/merge.go:18 +0xa7 fp=0xc0004d9a90 sp=0xc0004d9a30 pc=0x1454007 main.(*impl).IsActive(0xc00049af50, {0xc00007eeb0, 0x53e386}, 0xc00007eeb0) /go/src/github.com/kedacore/http-add-on/scaler/handlers.go:68 +0x145 fp=0xc0004d9b70 sp=0xc0004d9a90 pc=0x14513e5 github.com/kedacore/http-add-on/proto._ExternalScaler_IsActive_Handler({0x16a8ac0, 0xc00049af50}, {0x1a1c0e0, 0xc00002f590}, 0xc0001f9bc0, 0x0) /go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:139 +0x170 fp=0xc0004d9bc8 sp=0xc0004d9b70 pc=0x1420cf0 google.golang.org/grpc.(*Server).processUnaryRPC(0xc00013a8c0, {0x1a47a10, 0xc00031d1e0}, 0xc00002b200, 0xc0004a3770, 0x265be40, 0x0) /go/pkg/mod/google.golang.org/[email protected]/server.go:1282 +0xccf fp=0xc0004d9e48 sp=0xc0004d9bc8 pc=0x140e0cf google.golang.org/grpc.(*Server).handleStream(0xc00013a8c0, {0x1a47a10, 0xc00031d1e0}, 0xc00002b200, 0x0) /go/pkg/mod/google.golang.org/[email protected]/server.go:1616 +0xa2a fp=0xc0004d9f68 sp=0xc0004d9e48 pc=0x1411eaa google.golang.org/grpc.(*Server).serveStreams.func1.2() /go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98 fp=0xc0004d9fe0 sp=0xc0004d9f68 pc=0x140bbf8 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc0004d9fe8 sp=0xc0004d9fe0 pc=0x463081 created by google.golang.org/grpc.(*Server).serveStreams.func1 /go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 1 [semacquire]: sync.runtime_Semacquire(0xc0000001a0) /usr/local/go/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc00043e270) /usr/local/go/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc0004a32f0) /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:40 +0x27 main.main() /go/src/github.com/kedacore/http-add-on/scaler/main.go:130 +0x8c7

goroutine 6 [chan receive]: k8s.io/klog/v2.(*loggingT).flushDaemon(0x0) /go/pkg/mod/k8s.io/klog/[email protected]/klog.go:1181 +0x6a created by k8s.io/klog/v2.init.0 /go/pkg/mod/k8s.io/klog/[email protected]/klog.go:420 +0xfb

goroutine 7 [sleep]: time.Sleep(0x6fc23ac00) /usr/local/go/src/runtime/time.go:193 +0x12e sigs.k8s.io/controller-runtime/pkg/log.init.0.func1() /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/log/log.go:63 +0x38 created by sigs.k8s.io/controller-runtime/pkg/log.init.0 /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/log/log.go:62 +0x25

goroutine 24 [IO wait]: internal/poll.runtime_pollWait(0x7fece60a6c00, 0x72) /usr/local/go/src/runtime/netpoll.go:229 +0x89 internal/poll.(*pollDesc).wait(0xc000096380, 0xc000050800, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc000096380) /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc000096380) /usr/local/go/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc00043e330) /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc00043e330) /usr/local/go/src/net/tcpsock.go:262 +0x3d google.golang.org/grpc.(*Server).Serve(0xc00013a8c0, {0x1a18760, 0xc00043e330}) /go/pkg/mod/google.golang.org/[email protected]/server.go:779 +0x362 main.startGrpcServer({0x1a1c038, 0xc0004690c0}, {{0x1a39b78, 0xc00043b260}, 0x100010000}, 0x0, 0xc00043c180, 0xc00049ebe0, 0x64, 0xc8) /go/src/github.com/kedacore/http-add-on/scaler/main.go:166 +0x36e main.main.func3() /go/src/github.com/kedacore/http-add-on/scaler/main.go:97 +0x73 golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 23 [chan receive]: main.(*queuePinger).start(0xc00043c180, {0x1a1c038, 0xc0004690c0}, 0xc00049af00) /go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:77 +0x145 main.main.func2() /go/src/github.com/kedacore/http-add-on/scaler/main.go:89 +0x75 golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 11 [IO wait]: internal/poll.runtime_pollWait(0x7fece60a6eb8, 0x72) /usr/local/go/src/runtime/netpoll.go:229 +0x89 internal/poll.(*pollDesc).wait(0xc000096100, 0xc00032a000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000096100, {0xc00032a000, 0xeccc, 0xeccc}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc000096100, {0xc00032a000, 0xc00032a048, 0x6f5}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000424000, {0xc00032a000, 0x416bc6, 0xc0000a37f0}) /usr/local/go/src/net/net.go:183 +0x45 crypto/tls.(*atLeastReader).Read(0xc00008a120, {0xc00032a000, 0x0, 0x4099cd}) /usr/local/go/src/crypto/tls/conn.go:777 +0x3d bytes.(*Buffer).ReadFrom(0xc0004965f8, {0x19ec920, 0xc00008a120}) /usr/local/go/src/bytes/buffer.go:204 +0x98 crypto/tls.(*Conn).readFromUntil(0xc000496380, {0x19ef6e0, 0xc000424000}, 0xec89) /usr/local/go/src/crypto/tls/conn.go:799 +0xe5 crypto/tls.(*Conn).readRecordOrCCS(0xc000496380, 0x0) /usr/local/go/src/crypto/tls/conn.go:606 +0x112 crypto/tls.(*Conn).readRecord(...) /usr/local/go/src/crypto/tls/conn.go:574 crypto/tls.(*Conn).Read(0xc000496380, {0xc000186000, 0x1000, 0x920e00}) /usr/local/go/src/crypto/tls/conn.go:1277 +0x16f bufio.(*Reader).Read(0xc000089a40, {0xc00019e2e0, 0x9, 0x92edc2}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x19ec7c0, 0xc000089a40}, {0xc00019e2e0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00019e2e0, 0x9, 0xc00102eff0}, {0x19ec7c0, 0xc000089a40}) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00019e2a0) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95 golang.org/x/net/http2.(*clientConnReadLoop).run(0xc0000a3f98) /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130 golang.org/x/net/http2.(*ClientConn).readLoop(0xc00018e780) /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f created by golang.org/x/net/http2.(*Transport).newClientConn /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 22 [chan receive]: k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc000469080) /go/pkg/mod/k8s.io/[email protected]/pkg/watch/mux.go:247 +0x49 created by k8s.io/apimachinery/pkg/watch.NewBroadcaster /go/pkg/mod/k8s.io/[email protected]/pkg/watch/mux.go:73 +0x11b

goroutine 25 [semacquire]: sync.runtime_Semacquire(0xc00031c4e0) /usr/local/go/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc00043e3a8) /usr/local/go/src/sync/waitgroup.go:130 +0x71 golang.org/x/sync/errgroup.(*Group).Wait(0xc0004a3950) /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:40 +0x27 github.com/kedacore/http-add-on/pkg/routing.StartConfigMapRoutingTableUpdater({0x1a1c038, 0xc0004690c0}, {{0x1a39b78, 0xc00043b260}, 0x53e901}, 0xc0004a3170, {0xc000046028, 0x7}, 0xc00049ebe0, 0xc00043e1c8) /go/src/github.com/kedacore/http-add-on/pkg/routing/config_map_updater.go:79 +0x3c6 main.main.func4() /go/src/github.com/kedacore/http-add-on/scaler/main.go:110 +0x78 golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 20 [IO wait]: internal/poll.runtime_pollWait(0x7fece60a6dd0, 0x72) /usr/local/go/src/runtime/netpoll.go:229 +0x89 internal/poll.(*pollDesc).wait(0xc00043c500, 0xc0000b8000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00043c500, {0xc0000b8000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00043c500, {0xc0000b8000, 0x436387, 0xc0000aec30}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc000424008, {0xc0000b8000, 0xe, 0xc000026340}) /usr/local/go/src/net/net.go:183 +0x45 net/http.(*persistConn).Read(0xc00009c120, {0xc0000b8000, 0xc000048300, 0xc0000aed30}) /usr/local/go/src/net/http/transport.go:1926 +0x4e bufio.(*Reader).fill(0xc000436960) /usr/local/go/src/bufio/bufio.go:101 +0x103 bufio.(*Reader).Peek(0xc000436960, 0x1) /usr/local/go/src/bufio/bufio.go:139 +0x5d net/http.(*persistConn).readLoop(0xc00009c120) /usr/local/go/src/net/http/transport.go:2087 +0x1ac created by net/http.(*Transport).dialConn /usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 21 [select]: net/http.(*persistConn).writeLoop(0xc00009c120) /usr/local/go/src/net/http/transport.go:2386 +0xfb created by net/http.(*Transport).dialConn /usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 26 [IO wait]: internal/poll.runtime_pollWait(0x7fece60a6ce8, 0x72) /usr/local/go/src/runtime/netpoll.go:229 +0x89 internal/poll.(*pollDesc).wait(0xc000096300, 0x2, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc000096300) /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc000096300) /usr/local/go/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc00043e300) /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc00043e300) /usr/local/go/src/net/tcpsock.go:262 +0x3d net/http.(*Server).Serve(0xc0000c8000, {0x1a18760, 0xc00043e300}) /usr/local/go/src/net/http/server.go:3001 +0x394 net/http.(*Server).ListenAndServe(0xc0000c8000) /usr/local/go/src/net/http/server.go:2930 +0x7d github.com/kedacore/http-add-on/pkg/http.ServeContext({0x1a1c038, 0xc0004690c0}, {0xc0004a08a0, 0xc}, {0x19ef780, 0xc000469100}) /go/src/github.com/kedacore/http-add-on/pkg/http/server.go:18 +0x112 main.startAdminServer({0x1a1c038, 0xc0004690c0}, {{0x1a39b78, 0xc00043b260}, 0x0}, 0xc0004372c0, 0x0, 0xc00043c180) /go/src/github.com/kedacore/http-add-on/scaler/main.go:217 +0x445 main.main.func5() /go/src/github.com/kedacore/http-add-on/scaler/main.go:122 +0x65 golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 27 [chan receive]: github.com/kedacore/http-add-on/pkg/http.ServeContext.func1() /go/src/github.com/kedacore/http-add-on/pkg/http/server.go:15 +0x3c created by github.com/kedacore/http-add-on/pkg/http.ServeContext /go/src/github.com/kedacore/http-add-on/pkg/http/server.go:14 +0x108

goroutine 28 [chan receive]: main.startGrpcServer.func1() /go/src/github.com/kedacore/http-add-on/scaler/main.go:163 +0x3b created by main.startGrpcServer /go/src/github.com/kedacore/http-add-on/scaler/main.go:162 +0x35a

goroutine 29 [chan receive]: k8s.io/apimachinery/pkg/watch.(*filteredWatch).loop(0xc00049ed00) /go/pkg/mod/k8s.io/[email protected]/pkg/watch/filter.go:65 +0xa5 created by k8s.io/apimachinery/pkg/watch.Filter /go/pkg/mod/k8s.io/[email protected]/pkg/watch/filter.go:42 +0xef

goroutine 30 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc00011c528, 0xa) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0xc00049f180) /usr/local/go/src/sync/cond.go:56 +0x8c k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00011c500, 0xc000440860) /go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:527 +0x233 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000190480) /go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fece60d6a00) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9bf6e8, {0x19eefc0, 0xc0004a39e0}, 0x1, 0xc000048d80) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0001904e8, 0x3b9aca00, 0x0, 0xc0, 0x7fece60f9da0) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*controller).Run(0xc000190480, 0xc000048d80) /go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00011c140, 0x0) /go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:414 +0x498 github.com/kedacore/http-add-on/pkg/k8s.(*InformerConfigMapUpdater).Start(0x0, {0x1a1c038, 0xc000469380}) /go/src/github.com/kedacore/http-add-on/pkg/k8s/config_map_cache_informer.go:36 +0x59 github.com/kedacore/http-add-on/pkg/routing.StartConfigMapRoutingTableUpdater.func1() /go/src/github.com/kedacore/http-add-on/pkg/routing/config_map_updater.go:42 +0x4e golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 31 [select]: github.com/kedacore/http-add-on/pkg/routing.StartConfigMapRoutingTableUpdater.func2() /go/src/github.com/kedacore/http-add-on/pkg/routing/config_map_updater.go:48 +0x15a golang.org/x/sync/errgroup.(*Group).Go.func1() /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67 created by golang.org/x/sync/errgroup.(*Group).Go /go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 54 [chan receive]: k8s.io/client-go/tools/cache.(*processorListener).run.func1() /go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:782 +0x49 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fece60d6a00) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0004b7f38, {0x19eefc0, 0xc0004a3c20}, 0x1, 0xc000049320) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x0, 0xc0004b7f88) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(...) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90 k8s.io/client-go/tools/cache.(*processorListener).run(0xc000096180) /go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:781 +0x6b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 49 [chan receive]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc00011a000, 0x0) /go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:638 +0x45 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 50 [chan receive]: k8s.io/client-go/tools/cache.(*controller).Run.func1() /go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28 created by k8s.io/client-go/tools/cache.(*controller).Run /go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 51 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0000c80e0, {0x0, 0x0, 0x26745a0}, {0x1a09aa8, 0xc000469e00}, 0xc0000a7d18, 0xc000437c80, 0xc000048d80) /go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:469 +0x1b6 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0000c80e0, 0xc000048d80) /go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:429 +0x696 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fece60d6a00) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000462200, {0x19eefa0, 0xc00049afa0}, 0x1, 0xc000048d80) /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0000c80e0, 0xc000048d80) /go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x1f8 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 56 [select]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2() /go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:374 +0x12d created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:368 +0x378

goroutine 57 [select]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc000268900, 0xc0003d5600) /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaa8 golang.org/x/net/http2.(*clientStream).doRequest(0x0, 0x0) /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 55 [select]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000096180) /go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:752 +0x156 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1() /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 44 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00031d1e0) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259 created by google.golang.org/grpc/internal/transport.NewServerTransport /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 59 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc000268948, 0x9) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x9203a6) /usr/local/go/src/sync/cond.go:56 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc000268930, {0xc000216601, 0x5ff, 0x5ff}) /go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0x0}, {0xc000216601, 0x2, 0xc0004ddcb0}) /go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85 encoding/json.(*Decoder).refill(0xc00034ac80) /usr/local/go/src/encoding/json/stream.go:165 +0x17f encoding/json.(*Decoder).readValue(0xc00034ac80) /usr/local/go/src/encoding/json/stream.go:140 +0xbb encoding/json.(*Decoder).Decode(0xc00034ac80, {0x1601ca0, 0xc00043fad0}) /usr/local/go/src/encoding/json/stream.go:63 +0x78 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0001fad50, {0xc000131400, 0x400, 0x400}) /go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc00049b1d0, 0xc000498c00, {0x1a09878, 0xc000307e80}) /go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00049f760) /go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000469e00) /go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x11c created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 45 [runnable]: syscall.Syscall(0x0, 0xa, 0xc0003ec000, 0x8000) /usr/local/go/src/syscall/asm_linux_amd64.s:20 +0x5 syscall.read(0xc00043c580, {0xc0003ec000, 0xc0000a9a00, 0xc0000b0e20}) /usr/local/go/src/syscall/zsyscall_linux_amd64.go:687 +0x4d syscall.Read(...) /usr/local/go/src/syscall/syscall_unix.go:189 internal/poll.ignoringEINTRIO(...) /usr/local/go/src/internal/poll/fd_unix.go:582 internal/poll.(*FD).Read(0xc00043c580, {0xc0003ec000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:163 +0x285 net.(*netFD).Read(0xc00043c580, {0xc0003ec000, 0x1000100000027, 0x270000008e}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc00000e618, {0xc0003ec000, 0xc00008a4e0, 0x0}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc00010d500, {0xc00019e3c0, 0x9, 0xc00008a4e0}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x19ec7c0, 0xc00010d500}, {0xc00019e3c0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc00019e3c0, 0x9, 0xc00003a120}, {0x19ec7c0, 0xc00010d500}) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc00019e380) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00031d1e0, 0xc0002fc298, 0xc00031c9c0) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc00013a8c0, {0x1a47a10, 0xc00031d1e0}) /go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() /go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn /go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 43 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00008c5f0, 0x1) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0001f83c0) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 103 [select]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00008d4a0, 0x1) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00010d080) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85 google.golang.org/grpc/internal/transport.NewServerTransport.func2() /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6 created by google.golang.org/grpc/internal/transport.NewServerTransport /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 104 [select]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00012d1e0) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259 created by google.golang.org/grpc/internal/transport.NewServerTransport /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 105 [IO wait]: internal/poll.runtime_pollWait(0x7fece60a6a30, 0x72) /usr/local/go/src/runtime/netpoll.go:229 +0x89 internal/poll.(*pollDesc).wait(0xc00043c380, 0xc00001e000, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc00043c380, {0xc00001e000, 0x8000, 0x8000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc00043c380, {0xc00001e000, 0x60100000000, 0x8}) /usr/local/go/src/net/fd_posix.go:56 +0x29 net.(*conn).Read(0xc00000e2f0, {0xc00001e000, 0x40cc94, 0xc00012c000}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc00010d020, {0xc0000c8200, 0x9, 0x101000000000018}) /usr/local/go/src/bufio/bufio.go:227 +0x1b4 io.ReadAtLeast({0x19ec7c0, 0xc00010d020}, {0xc0000c8200, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:328 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:347 golang.org/x/net/http2.readFrameHeader({0xc0000c8200, 0x9, 0xc00004b518}, {0x19ec7c0, 0xc00010d020}) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc0000c81c0) /go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00012d1e0, 0xc0002fc3b8, 0xc00012cb60) /go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2 google.golang.org/grpc.(*Server).serveStreams(0xc00013a8c0, {0x1a47a10, 0xc00012d1e0}) /go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142 google.golang.org/grpc.(*Server).handleRawConn.func1() /go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46 created by google.golang.org/grpc.(*Server).handleRawConn /go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185 `

Expected Behavior

Keda http-add-on should just work, now the external-scaler is in a CrashLoopBackOff

Actual Behavior

When trying to deploy a private service with keda http addon , we ran into the issiue that we do not want to create an ingress, how we resolved it is by creating an ExternalName

Steps to Reproduce the Problem

Install a service and point to it using an ExternalName instead of an ingress

Logs from KEDA HTTP operator

example

What version of the KEDA HTTP Add-on are you running?

No response

Kubernetes Version

1.21

Platform

Amazon Web Services

Anything else?

No response

Sep 07 '22 08:09 abhishektvz

very similar issue, except it says fatal error: concurrent map read and map write

Log: keda-add-ons-http-external-scaler-5575f94db-5z2xt.log

After some crashloops it seems it eventually starts without error. I have 2 Pods connected with it to scale from zero using ingress.

Kubernetes Version: 1.22.13

Sep 26 '22 07:09 lsascha

Same issue with @lsascha fatal error: concurrent map read and map write Full log: https://pastebin.com/raw/NCaf3zhK

I have ~ 3 deployments scaled to zero using Ingress. I am struggling to keep http-external-scaler up and running, when external scaler is down then all scaled to zero deployments wont scale up.

Oct 18 '22 11:10 christidis

Same issue here. fatal error: concurrent map read and map write Full log: https://pastebin.com/raw/JWEBKZ2N

2 deployments, using alb ingress. keda-add-ons-http-external-scaler is in CrashLoopBackOff over and over..

Nov 21 '22 10:11 Iliabuleh

I think this is more easily reproducible when having more than just one Deployment scaled with keda-http. (2 or 3+ HTTPScaledObject deployed).

Dec 06 '22 16:12 christidis

We started seeing this as well as soon as we deployed more than 1 HTTPScaledObject.

Seeing:

{"level":"info","ts":1670960133.3454225,"logger":"startHealthcheckServer","caller":"scaler/main.go:216","msg":"starting health check server","addr":"0.0.0.0:9091"}
{"level":"info","ts":1670960133.3454528,"caller":"scaler/main.go:144","msg":"starting grpc server","address":"0.0.0.0:9090"}
fatal error: concurrent map writes

goroutine 4580 [running]:
runtime.throw({0x180ee20, 0x8010101})
	/usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc000b9b978 sp=0xc000b9b948 pc=0x433831
runtime.mapassign_faststr(0x15de900, 0xc0008441b0, {0xc000a8c030, 0x27})
	/usr/local/go/src/runtime/map_faststr.go:294 +0x38b fp=0xc000b9b9e0 sp=0xc000b9b978 pc=0x412a8b
main.mergeCountsWithRoutingTable(0xc000144200, {0x1a068a8, 0xc00014c4a0})
	/go/src/github.com/kedacore/http-add-on/scaler/merge.go:18 +0xa7 fp=0xc000b9ba40 sp=0xc000b9b9e0 pc=0x1454007
main.(*impl).IsActive(0xc000134640, {0xc000b9bb78, 0x203000}, 0xc0001806e0)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:68 +0x145 fp=0xc000b9bb20 sp=0xc000b9ba40 pc=0x14513e5
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc0002764d0})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a fp=0xc000b9bbf0 sp=0xc000b9bb20 pc=0x145184a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc0002d2000})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0 fp=0xc000b9bc30 sp=0xc000b9bbf0 pc=0x1420f30
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc000317380}, 0xc00055f320, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba fp=0xc000b9be48 sp=0xc000b9bc30 pc=0x141097a
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc000317380}, 0xc00055f320, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5 fp=0xc000b9bf68 sp=0xc000b9be48 pc=0x1411e65
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98 fp=0xc000b9bfe0 sp=0xc000b9bf68 pc=0x140bbf8
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc000b9bfe8 sp=0xc000b9bfe0 pc=0x463081
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 1 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc0000001a0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0001285e8)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
golang.org/x/sync/errgroup.(*Group).Wait(0xc000532f60)
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:40 +0x27
main.main()
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:130 +0x8c7

goroutine 18 [chan receive]:
k8s.io/klog/v2.(*loggingT).flushDaemon(0x0)
	/go/pkg/mod/k8s.io/klog/[email protected]/klog.go:1181 +0x6a
created by k8s.io/klog/v2.init.0
	/go/pkg/mod/k8s.io/klog/[email protected]/klog.go:420 +0xfb

goroutine 4567 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7778, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4300, 0xc000bee000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4300, {0xc000bee000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000e4300, {0xc000bee000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc050, {0xc000bee000, 0x40cc94, 0x7f5a8fdb0b78})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0003f6900, {0xc0005bcac0, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc0003f6900}, {0xc0005bcac0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005bcac0, 0x9, 0xc000e8ebe8}, {0x19ec7c0, 0xc0003f6900})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005bca80)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0003c5d40, 0x1155afae10, 0x26745a0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc0003c5d40})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 30 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7c00, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522580, 0xc00005f000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000522580)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000522580)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000128690)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000128690)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0004ea540, {0x1a18760, 0xc000128690})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:779 +0x362
main.startGrpcServer({0x1a1c038, 0xc0001405c0}, {{0x1a39b78, 0xc0001301e0}, 0x0}, 0x0, 0xc000144200, 0xc00014c4a0, 0x64, 0xc8)
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:166 +0x36e
main.main.func3()
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:97 +0x73
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 10 [chan receive]:
k8s.io/apimachinery/pkg/watch.(*filteredWatch).loop(0xc0004b3ac0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/filter.go:65 +0xa5
created by k8s.io/apimachinery/pkg/watch.Filter
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/filter.go:42 +0xef

goroutine 24 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7eb8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000144380, 0xc0002c6000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000144380, {0xc0002c6000, 0x3366, 0x3366})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000144380, {0xc0002c6000, 0xc0002c6048, 0x6b3})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00011e010, {0xc0002c6000, 0x203000, 0xc00015d7f0})
	/usr/local/go/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0006fcd38, {0xc0002c6000, 0x0, 0x4099cd})
	/usr/local/go/src/crypto/tls/conn.go:777 +0x3d
bytes.(*Buffer).ReadFrom(0xc0002c4278, {0x19ec920, 0xc0006fcd38})
	/usr/local/go/src/bytes/buffer.go:204 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0002c4000, {0x19ef6e0, 0xc00011e010}, 0x3323)
	/usr/local/go/src/crypto/tls/conn.go:799 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0002c4000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:606 +0x112
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:574
crypto/tls.(*Conn).Read(0xc0002c4000, {0xc000226000, 0x1000, 0x920e00})
	/usr/local/go/src/crypto/tls/conn.go:1277 +0x16f
bufio.(*Reader).Read(0xc00008a300, {0xc0005bc200, 0x9, 0x92edc2})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc00008a300}, {0xc0005bc200, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005bc200, 0x9, 0xc001f7ce40}, {0x19ec7c0, 0xc00008a300})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005bc1c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00015df98)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2101 +0x130
golang.org/x/net/http2.(*ClientConn).readLoop(0xc0003f8180)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1997 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:725 +0xac5

goroutine 29 [chan receive]:
main.(*queuePinger).start(0xc000144200, {0x1a1c038, 0xc0001405c0}, 0xc000485090)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:77 +0x145
main.main.func2()
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:89 +0x75
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 28 [chan receive]:
k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc000140580)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/mux.go:247 +0x49
created by k8s.io/apimachinery/pkg/watch.NewBroadcaster
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/mux.go:73 +0x11b

goroutine 8 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7dd0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522380, 0xc0000a0000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000522380, {0xc0000a0000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000522380, {0xc0000a0000, 0x436387, 0xc00009dc30})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc060, {0xc0000a0000, 0xe, 0xc000acd6c0})
	/usr/local/go/src/net/net.go:183 +0x45
net/http.(*persistConn).Read(0xc0001e0360, {0xc0000a0000, 0xc000183140, 0xc00009dd30})
	/usr/local/go/src/net/http/transport.go:1926 +0x4e
bufio.(*Reader).fill(0xc0004de6c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).Peek(0xc0004de6c0, 0x1)
	/usr/local/go/src/bufio/bufio.go:139 +0x5d
net/http.(*persistConn).readLoop(0xc0001e0360)
	/usr/local/go/src/net/http/transport.go:2087 +0x1ac
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1747 +0x1e05

goroutine 9 [select]:
net/http.(*persistConn).writeLoop(0xc0001e0360)
	/usr/local/go/src/net/http/transport.go:2386 +0xfb
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1748 +0x1e65

goroutine 31 [semacquire, 1 minutes]:
sync.runtime_Semacquire(0xc000102ea0)
	/usr/local/go/src/runtime/sema.go:56 +0x25
sync.(*WaitGroup).Wait(0xc0004e6420)
	/usr/local/go/src/sync/waitgroup.go:130 +0x71
golang.org/x/sync/errgroup.(*Group).Wait(0xc000130d50)
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:40 +0x27
github.com/kedacore/http-add-on/pkg/routing.StartConfigMapRoutingTableUpdater({0x1a1c038, 0xc0001405c0}, {{0x1a39b78, 0xc0001301e0}, 0x0}, 0xc000532de0, {0xc000044088, 0x8}, 0xc00014c4a0, 0xc000128540)
	/go/src/github.com/kedacore/http-add-on/pkg/routing/config_map_updater.go:79 +0x3c6
main.main.func4()
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:110 +0x78
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 32 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f5a690d7ce8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522500, 0x2, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000522500)
	/usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc000522500)
	/usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc000128678)
	/usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc000128678)
	/usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc0005bc2a0, {0x1a18760, 0xc000128678})
	/usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc0005bc2a0)
	/usr/local/go/src/net/http/server.go:2930 +0x7d
github.com/kedacore/http-add-on/pkg/http.ServeContext({0x1a1c038, 0xc0001405c0}, {0xc00013c7d0, 0xc}, {0x19ef780, 0xc000140600})
	/go/src/github.com/kedacore/http-add-on/pkg/http/server.go:18 +0x112
main.startAdminServer({0x1a1c038, 0xc0001405c0}, {{0x1a39b78, 0xc0001301e0}, 0x0}, 0xc00013e1e0, 0x0, 0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:217 +0x445
main.main.func5()
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:122 +0x65
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 65 [chan receive, 1 minutes]:
github.com/kedacore/http-add-on/pkg/http.ServeContext.func1()
	/go/src/github.com/kedacore/http-add-on/pkg/http/server.go:15 +0x3c
created by github.com/kedacore/http-add-on/pkg/http.ServeContext
	/go/src/github.com/kedacore/http-add-on/pkg/http/server.go:14 +0x108

goroutine 11 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000136708, 0xaf)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0001b42a0)
	/usr/local/go/src/sync/cond.go:56 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001366e0, 0xc0004e9ac0)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:527 +0x233
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000151e60)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f5a69113888)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x9bf6e8, {0x19eefc0, 0xc000130de0}, 0x1, 0xc000183380)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000151ec8, 0x3b9aca00, 0x0, 0x80, 0x7f5a68f8c300)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc000151e60, 0xc000183380)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2fb
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0001b6500, 0xc8)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:414 +0x498
github.com/kedacore/http-add-on/pkg/k8s.(*InformerConfigMapUpdater).Start(0xc000144200, {0x1a1c038, 0xc000591080})
	/go/src/github.com/kedacore/http-add-on/pkg/k8s/config_map_cache_informer.go:36 +0x59
github.com/kedacore/http-add-on/pkg/routing.StartConfigMapRoutingTableUpdater.func1()
	/go/src/github.com/kedacore/http-add-on/pkg/routing/config_map_updater.go:42 +0x4e
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 12 [select, 1 minutes]:
github.com/kedacore/http-add-on/pkg/routing.StartConfigMapRoutingTableUpdater.func2()
	/go/src/github.com/kedacore/http-add-on/pkg/routing/config_map_updater.go:48 +0x15a
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:57 +0x67
created by golang.org/x/sync/errgroup.(*Group).Go
	/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:54 +0x92

goroutine 53 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:782 +0x49
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f5a690debb8)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00051c738, {0x19eefc0, 0xc000251320}, 0x1, 0xc0000240c0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x80, 0xc00051c788)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc000522400)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:781 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 14 [chan receive, 1 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0001b0cb0, 0xc000048960)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:638 +0x45
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 15 [chan receive, 1 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x105

goroutine 54 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc000522400)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/shared_informer.go:752 +0x156
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 16 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc00023a2a0, {0x0, 0x0, 0x26745a0}, {0x1a09aa8, 0xc00045c0c0}, 0xc000157d18, 0xc00008a1e0, 0xc000183380)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:469 +0x1b6
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00023a2a0, 0xc000183380)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:429 +0x696
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f5a69113888)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000422700, {0x19eefa0, 0xc0004850e0}, 0x1, 0xc000183380)
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00023a2a0, 0xc000183380)
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x1f8
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x88

goroutine 66 [chan receive, 1 minutes]:
main.startGrpcServer.func1()
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:163 +0x3b
created by main.startGrpcServer
	/go/src/github.com/kedacore/http-add-on/scaler/main.go:162 +0x35a

goroutine 83 [select, 1 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2()
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:374 +0x12d
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
	/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:368 +0x378

goroutine 84 [select, 1 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc000565b00, 0xc00027c400)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1323 +0xaa8
golang.org/x/net/http2.(*clientStream).doRequest(0xc0001b0d10, 0xc00051c7b8)
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1185 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1114 +0x30f

goroutine 4566 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0003c5d40)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 67 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000565b48, 0xaf)
	/usr/local/go/src/runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xd)
	/usr/local/go/src/sync/cond.go:56 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc000565b30, {0xc0002b7801, 0x5ff, 0x5ff})
	/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x0}, {0xc0002b7801, 0x2, 0xc000511cb0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2384 +0x85
encoding/json.(*Decoder).refill(0xc0005e8140)
	/usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc0005e8140)
	/usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc0005e8140, {0x1601ca0, 0xc000b1c750})
	/usr/local/go/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00045a210, {0xc00014ec00, 0x400, 0x400})
	/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x19c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0001340f0, 0xc0001e8300, {0x1a09878, 0xc0009298c0})
	/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00014c040)
	/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00045c0c0)
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:105 +0x11c
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
	/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:76 +0x135

goroutine 4437 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x0)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc00010db78, 0x203000}, 0xc000707130)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000455900})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc0000fc240})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc0003c5d40}, 0xc00037c900, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc0003c5d40}, 0xc00037c900, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4600 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7a30, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4200, 0xc000bc6000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4200, {0xc000bc6000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000e4200, {0xc000bc6000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc040, {0xc000bc6000, 0x40cc94, 0xc0002589c0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000460ba0, {0xc000994900, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc000460ba0}, {0xc000994900, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000994900, 0x9, 0xc000a3eb28}, {0x19ec7c0, 0xc000460ba0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009948c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006ff6c0, 0x13af085, 0x13ac9ef)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc0006ff6c0})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4602 [runnable]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000134cd0, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004613e0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4442 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a964b0, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003ecde0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4440 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000bb380)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4601 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x1)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc0000f7b78, 0x203000}, 0xc000134af0)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000531d90})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc000293380})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc0006ff6c0}, 0xc0005e7b00, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc0006ff6c0}, 0xc0005e7b00, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4604 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7690, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4400, 0xc000c36000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4400, {0xc000c36000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000e4400, {0xc000c36000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc088, {0xc000c36000, 0x40cc94, 0xc00094a1a0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000461380, {0xc0009949e0, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc000461380}, {0xc0009949e0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0009949e0, 0x9, 0xc000e7e990}, {0x19ec7c0, 0xc000461380})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009949a0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006ff860, 0x13af085, 0x13ac9ef)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc0006ff860})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4509 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000317380)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4605 [select]:
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000531f70})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:94 +0x113
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc000293440})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc0006ff860}, 0xc0005e7d40, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc0006ff860}, 0xc0005e7d40, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4510 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d72f0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522a00, 0xc000c80000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000522a00, {0xc000c80000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000522a00, {0xc000c80000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000198068, {0xc000c80000, 0x40cc94, 0xc000859a00})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00013e3c0, {0xc00023a740, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc00013e3c0}, {0xc00023a740, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00023a740, 0x9, 0xc000a3eae0}, {0x19ec7c0, 0xc00013e3c0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023a700)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000317380, 0x11eb649732, 0x26745a0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc000317380})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4445 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a96690, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003ed500)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4596 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7b18, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4100, 0xc000bae000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4100, {0xc000bae000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000e4100, {0xc000bae000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc038, {0xc000bae000, 0x40cc94, 0xc000a52000})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000460b40, {0xc000994820, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc000460b40}, {0xc000994820, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000994820, 0x9, 0xc000a3eb10}, {0x19ec7c0, 0xc000460b40})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0009947e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006ff520, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc0006ff520})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4609 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x0)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc000c61b78, 0x203000}, 0xc000a96aa0)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000ae2730})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc000af4180})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc00049a1a0}, 0xc00046b560, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc00049a1a0}, 0xc00046b560, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4447 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d73d8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522980, 0xc000cc8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000522980, {0xc000cc8000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000522980, {0xc000cc8000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000198060, {0xc000cc8000, 0x40cc94, 0xc000a531e0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0003ed260, {0xc00048a200, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc0003ed260}, {0xc00048a200, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00048a200, 0x9, 0xc0001c57a0}, {0x19ec7c0, 0xc0003ed260})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00048a1c0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00049a1a0, 0xc0008ce658, 0xc0004f9d40)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc00049a1a0})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4446 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00049a1a0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4598 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001344b0, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004610e0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4441 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d75a8, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522880, 0xc000c68000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000522880, {0xc000c68000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000522880, {0xc000c68000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000198050, {0xc000c68000, 0x40cc94, 0xc00094a9c0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0003eca20, {0xc00048a040, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc0003eca20}, {0xc00048a040, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00048a040, 0x9, 0xc000a3eb40}, {0x19ec7c0, 0xc0003eca20})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00048a000)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000bb380, 0x13af085, 0x13ac9ef)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc0000bb380})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4443 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00049a000)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4444 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d74c0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc000522900, 0xc000ca8000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000522900, {0xc000ca8000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000522900, {0xc000ca8000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000198058, {0xc000ca8000, 0x40cc94, 0xc00094b6c0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0003ecd80, {0xc00048a120, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc0003ecd80}, {0xc00048a120, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00048a120, 0x9, 0xc0001c5788}, {0x19ec7c0, 0xc0003ecd80})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00048a0e0)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00049a000, 0x13af085, 0x13ac9ef)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc00049a000})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4438 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x0)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc000c65b78, 0x203000}, 0xc000a961e0)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000ae2190})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc000af4000})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc000f6d520}, 0xc00037de60, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc000f6d520}, 0xc00037de60, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4560 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006ff380)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4594 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001342d0, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000460d80)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4593 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7948, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4280, 0xc000b8e000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4280, {0xc000b8e000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000e4280, {0xc000b8e000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc048, {0xc000b8e000, 0x40cc94, 0xc0003c5520})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000460ae0, {0xc000994740, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc000460ae0}, {0xc000994740, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc000994740, 0x9, 0xc000a3eaf8}, {0x19ec7c0, 0xc000460ae0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000994700)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006ff380, 0x0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc0006ff380})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4569 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000f6d520)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4565 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000706c30, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003f6960)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4439 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000a962d0, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003eca80)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4568 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000706e10, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003f6d80)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4599 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006ff6c0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4571 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x0)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc0004a9b78, 0x203000}, 0xc000707090)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc0004557d0})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc0000fc180})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc0006ff380}, 0xc000ab1560, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc0006ff380}, 0xc000ab1560, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4448 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x0)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc000c67b78, 0x203000}, 0xc000a96910)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000ae2510})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc000af40c0})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc0000bb380}, 0xc00046a240, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc0000bb380}, 0xc00046a240, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4603 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006ff860)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

goroutine 4508 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000095e00, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0002944e0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4625 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x1)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc000d15b78, 0x203000}, 0xc0007040f0)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc00003a510})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc00085a000})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc00049a000}, 0xc00035c7e0, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc00049a000}, 0xc00035c7e0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4570 [IO wait]:
internal/poll.runtime_pollWait(0x7f5a690d7860, 0x72)
	/usr/local/go/src/runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4380, 0xc000c0e000, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4380, {0xc000c0e000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0000e4380, {0xc000c0e000, 0x60100000000, 0x8})
	/usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004cc068, {0xc000c0e000, 0x40cc94, 0xc0000bb1e0})
	/usr/local/go/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0003f6ae0, {0xc0005bcba0, 0x9, 0x101000000000018})
	/usr/local/go/src/bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x19ec7c0, 0xc0003f6ae0}, {0xc0005bcba0, 0x9, 0x9}, 0x9)
	/usr/local/go/src/io/io.go:328 +0x9a
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005bcba0, 0x9, 0xc0001c5770}, {0x19ec7c0, 0xc0003f6ae0})
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005bcb60)
	/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000f6d520, 0x1155afae10, 0x26745a0)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:605 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc0004ea540, {0x1a47a10, 0xc000f6d520})
	/go/pkg/mod/google.golang.org/[email protected]/server.go:905 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:847 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
	/go/pkg/mod/google.golang.org/[email protected]/server.go:846 +0x185

goroutine 4597 [runnable]:
sync.runtime_SemacquireMutex(0x171ed40, 0x1, 0x1)
	/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*RWMutex).RLock(...)
	/usr/local/go/src/sync/rwmutex.go:63
main.(*queuePinger).counts(0xc000144200)
	/go/src/github.com/kedacore/http-add-on/scaler/queue_pinger.go:103 +0x58
main.(*impl).IsActive(0xc000134640, {0xc0000f5b78, 0x203000}, 0xc000134960)
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:69 +0x130
main.(*impl).StreamIsActive(0xc000134640, 0x16f9b20, {0x1a3c230, 0xc000531c40})
	/go/src/github.com/kedacore/http-add-on/scaler/handlers.go:98 +0x14a
github.com/kedacore/http-add-on/proto._ExternalScaler_StreamIsActive_Handler({0x16a8ac0, 0xc000134640}, {0x1a39c98, 0xc000292b40})
	/go/src/github.com/kedacore/http-add-on/proto/scaler_grpc.pb.go:156 +0xd0
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0004ea540, {0x1a47a10, 0xc0006ff520}, 0xc0005e77a0, 0xc000533350, 0x2653de0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1540 +0xeba
google.golang.org/grpc.(*Server).handleStream(0xc0004ea540, {0x1a47a10, 0xc0006ff520}, 0xc0005e77a0, 0x0)
	/go/pkg/mod/google.golang.org/[email protected]/server.go:1620 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
	/go/pkg/mod/google.golang.org/[email protected]/server.go:921 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/go/pkg/mod/google.golang.org/[email protected]/server.go:919 +0x294

goroutine 4559 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000134190, 0x1)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:407 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000460c00)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/controlbuf.go:534 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:326 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:323 +0x180f

goroutine 4595 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006ff520)
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:1125 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
	/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:335 +0x1857

Anyone of the HTTP Keda add-on maintainers that can chime in here and advise? Is any work being done to fix this and is there a workaround? This is pretty serious as the functionality is very sporadic in regards to the KEDA HTTP Add-on being able to scale u workloads.

Thank you very much.

Dec 13 '22 19:12 LarsBingBong

Any news on this one. Hitting this issue all the time. So it pretty much - unfortunately so - renders the http-add-on useless when this bug is hit. I really hope that this will be looked at some point soon 🤞

Thank you very much ...

Jan 21 '23 21:01 larssb

Could this issue be duplicated? https://github.com/kedacore/http-add-on/issues/414 I can't find the code in the stack trace and I have found this https://github.com/kedacore/http-add-on/pull/415

Jan 24 '23 18:01 JorTurFer

This bug was fixed and the fix has been released as v0.4.0

Jan 27 '23 16:01 JorTurFer

http-add-on http-add-on copied to clipboard

fatal error: concurrent map writes

Report

Expected Behavior

Actual Behavior

Steps to Reproduce the Problem

Logs from KEDA HTTP operator

What version of the KEDA HTTP Add-on are you running?

Kubernetes Version

Platform

Anything else?

http-add-on
http-add-on copied to clipboard