gitlab-ci-pipelines-exporter
gitlab-ci-pipelines-exporter copied to clipboard
Published
20 hours ago •
mvisonneau
mvisonneau
SIGSEGV in redis taskq
Hello,
About an hour into the run, the exporter crashes with a SIGSEGV. We're using the exporter on a single monorepo project with a lot of activity.
Version: quay.io/mvisonneau/gitlab-ci-pipelines-exporter:v0.5.4 Installation: Helm on AWS EKS
replicas: 1
resources:
requests:
cpu: 4000m
memory: 8192Mi
limits:
cpu: 4000m
memory: 8192Mi
serviceMonitor:
enabled: true
ingress:
enabled: true
annotations:
alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "true"
hosts:
- ***
tls:
- hosts:
- ***
ingressClassName: alb
config:
log:
level: debug
server:
webhook:
enabled: true
gitlab:
url: ***
maximum_requests_per_second: 10
garbage_collect:
projects:
on_init: true
environments:
on_init: true
refs:
on_init: true
metrics:
on_init: true
project_defaults:
pull:
refs:
branches:
enabled: true
regexp: ".*"
max_age_seconds: 86400 # 24 hours
merge_requests:
enabled: true
max_age_seconds: 86400 # 24 hours
pipeline:
jobs:
enabled: true
from_child_pipelines:
enabled: true
projects:
- name: ***
pull:
refs:
branches:
enabled: true
regexp: ".*"
max_age_seconds: 86400 # 24 hours
tags:
enabled: false
merge_requests:
enabled: true
max_age_seconds: 86400 # 24 hours
pipeline:
jobs:
enabled: true
from_child_pipelines:
enabled: true
variables:
enabled: true
redis:
master:
resources:
requests:
cpu: 500m
memory: 8Gi
limits:
cpu: 1000m
memory: 8Gi
metrics:
enabled: true
serviceMonitor:
enabled: true
Logs:
{"ip-address":"10.208.73.19:26882","level":"debug","msg":"webhook request","time":"2023-02-16T23:15:07Z","user-agent":"GitLab/15.4.6-ee"}
{"error":"context canceled","level":"error","msg":"reading ref from the store","project-name":"***","ref":"***","ref-kind":"branch","time":"2023-02-16T23:15:07Z"}
{"for":36135158,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:07Z"}
{"for":50287475,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:07Z"}
{"for":18245722,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:07Z"}
{"for":65370979,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:08Z"}
{"for":20137077,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:08Z"}
{"for":42851310,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:08Z"}
{"for":12097230,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:08Z"}
{"for":55173424,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:08Z"}
{"for":44198664,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:09Z"}
{"for":56197403,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:09Z"}
{"for":20699617,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:09Z"}
{"for":42131286,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:09Z"}
{"for":76813949,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:10Z"}
{"for":17027181,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:10Z"}
{"jobs-count":3,"level":"debug","msg":"found all jobs to refresh","project-name":"***","ref":"***","time":"2023-02-16T23:15:11Z"}
{"for":38951829,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:11Z"}
{"for":36151258,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:11Z"}
{"for":13047887,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:11Z"}
{"for":82305719,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:11Z"}
{"for":41157131,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:11Z"}
{"for":64221740,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:11Z"}
{"for":25959901,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:12Z"}
{"for":38416339,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:12Z"}
{"for":22192086,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:12Z"}
{"for":38136505,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:12Z"}
{"for":12972956,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:13Z"}
{"for":74794764,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:13Z"}
{"for":39085721,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:13Z"}
{"for":13649928,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:14Z"}
{"for":29685128,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:14Z"}
{"for":43041340,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:14Z"}
{"for":14178187,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":72628636,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":86928395,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":59264283,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":47886835,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":11197926,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":60434744,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:15Z"}
{"for":16414936,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":12094131,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":69919329,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":35899856,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":21615737,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":62337867,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":95163469,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":58316106,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:16Z"}
{"for":20144242,"level":"debug","msg":"throttled GitLab requests","time":"2023-02-16T23:15:17Z"}
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0x9c7490]
goroutine 93533 [running]:
github.com/go-redis/redis/v8/internal/pool.(*ConnPool).waitTurn(0xc00056d7c0, {0x0, 0x0})
github.com/go-redis/redis/[email protected]/internal/pool/pool.go:292 +0x30
github.com/go-redis/redis/v8/internal/pool.(*ConnPool).Get(0xc00056d7c0, {0x0, 0x0})
github.com/go-redis/redis/[email protected]/internal/pool/pool.go:249 +0x45
github.com/go-redis/redis/v8.(*baseClient)._getConn(0xc0001a1080, {0x0, 0x0})
github.com/go-redis/redis/[email protected]/redis.go:194 +0x38
github.com/go-redis/redis/v8.(*baseClient).getConn(0xc0001a1080, {0x0?, 0x0?})
github.com/go-redis/redis/[email protected]/redis.go:182 +0x69
github.com/go-redis/redis/v8.(*baseClient).withConn(0xc0001a1080, {0x0?, 0x0}, 0xc01428aa80)
github.com/go-redis/redis/[email protected]/redis.go:274 +0x73
github.com/go-redis/redis/v8.(*baseClient)._generalProcessPipeline(0xc0001a1080, {0x0, 0x0}, {0xc01e4cacd0, 0x5, 0x5}, 0xc00a7dcce0)
github.com/go-redis/redis/[email protected]/redis.go:424 +0x17c
github.com/go-redis/redis/v8.(*baseClient).generalProcessPipeline(0x203004?, {0x0?, 0x0?}, {0xc01e4cacd0, 0x5, 0x5}, 0x0?)
github.com/go-redis/redis/[email protected]/redis.go:404 +0x3f
github.com/go-redis/redis/v8.(*baseClient).processTxPipeline(0xc0001a1080, {0x0, 0x0}, {0xc01e4cacd0, 0x5, 0x5})
github.com/go-redis/redis/[email protected]/redis.go:396 +0x95
github.com/go-redis/redis/v8.hooks.processPipeline({{0xc00058d900?, 0xc01e4cace0?, 0x203000?}}, {0x0?, 0x0?}, {0xc01e4cacd0, 0x5, 0x5}, 0xc013163cb8)
github.com/go-redis/redis/[email protected]/redis.go:103 +0x194
github.com/go-redis/redis/v8.hooks.processTxPipeline({{0xc00058d900?, 0x4000105?, 0x0?}}, {0x0?, 0x0}, {0xc014298580, 0x3, 0x0?}, 0x0?)
github.com/go-redis/redis/[email protected]/redis.go:120 +0x2db
github.com/go-redis/redis/v8.(*Client).processTxPipeline(0x0?, {0x0?, 0x0?}, {0xc014298580?, 0x0?, 0x7f7f0cb7f538?})
github.com/go-redis/redis/[email protected]/redis.go:604 +0x6e
github.com/go-redis/redis/v8.(*Pipeline).Exec(0xc00033a000?, {0x0?, 0x0?})
github.com/go-redis/redis/[email protected]/pipeline.go:125 +0x15f
github.com/vmihailenco/taskq/v3/redisq.(*Queue).Release(0xc00033a000, 0xc01b5fa000)
github.com/vmihailenco/taskq/[email protected]/redisq/queue.go:245 +0x1ae
github.com/vmihailenco/taskq/v3.(*Consumer).fetchMessages(0xc0001252b0, {0x10de9e8, 0xc000134008}, 0xc01164ec80, 0x0?)
github.com/vmihailenco/taskq/[email protected]/consumer.go:439 +0x291
github.com/vmihailenco/taskq/v3.(*Consumer).fetcher(0xc0001252b0, {0x10de9e8, 0xc000134008}, 0x1)
github.com/vmihailenco/taskq/[email protected]/consumer.go:394 +0x1a5
github.com/vmihailenco/taskq/v3.(*Consumer).addFetcher.func1()
github.com/vmihailenco/taskq/[email protected]/consumer.go:295 +0x69
created by github.com/vmihailenco/taskq/v3.(*Consumer).addFetcher
github.com/vmihailenco/taskq/[email protected]/consumer.go:293 +0x148
