Add more debug info for "invalid memory address or nil pointer dereference"

[frimuchkov]

Hi we get 3-4 errors in WF per day:

runtime error: invalid memory address or nil pointer dereference

goroutine 366 [running]:
runtime/debug.Stack()
   /usr/local/go/src/runtime/debug/stack.go:24 +0x65
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).operate.func2()
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:194 +0xd4
panic({0x1d506c0, 0x3431820})
   /usr/local/go/src/runtime/panic.go:1047 +0x266
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc016a6b340, 0x146}, {0x231a270, 0xc0166ffef0}, 0x1, {{0xc016ab1080, 0x3, ...}, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1948 +0x32c5
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eba40, {0xc0165a7401, 0x2c})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1888
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eba40, {0xc016121881, 0x9})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:438 +0x1f25
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eba40, {0xc0165ea601, 0x10})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:438 +0x1f25
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eba40, {0xc0165fe480, 0x10})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:438 +0x1f25
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00e0b9d40, 0x6d49364969626c00}, 0xc0084107c0, {0xc015f65290, 0x29}, 0xc01638cfc0, {0x231a270, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0x433
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00e0b9d40, 0x119}, {0x231a270, 0xc015f59590}, 0x0, {{0xc016388000, 0x15, ...}, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1889 +0x232c
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eaf50, {0xc008de75c0, 0x36})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1888
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc0060c45a0, 0x3053465247523400}, 0xc0072c0780, {0xc00bd7bf20, 0x29}, 0xc00dac3b00, {0x231b500, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0x433
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc0060c45a0, 0xe2}, {0x231b500, 0xc00072d380}, 0x5a6977694979566d, {{0xc00dada2c0, 0x3, ...}, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1889 +0x232c
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeStepGroup(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00c8cc900, 0x1, 0x4}, {0xc00dac5500, 0x62}, 0xc00e0a0110)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/steps.go:247 +0x606
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeSteps(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00dabe1e0, 0x5f}, 0xc006ebc6c0, {0xc00bd7a7b0, 0x29}, 0xc00dac26c0, {0x231a270, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/steps.go:95 +0xd05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00dabe1e0, 0x5f}, {0x231a270, 0xc0084c7e00}, 0x0, {{0x0, 0x0, ...}, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1883 +0x246c
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6ead20, {0xc005eb4540, 0x1a})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1888
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00b686730, 0x6977536639316e00}, 0xc006ebc2c0, {0xc00bd7a5a0, 0x29}, 0xc00daab680, {0x231a270, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0x433
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00b686730, 0x44}, {0x231a270, 0xc0084c7770}, 0x0, {{0x0, 0x0, ...}, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1889 +0x232c
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eab60, {0xc00d7a8701, 0x20})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1888
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00072d800, {0x2339608, 0xc00005e018}, 0xc00b6eab60, {0xc0050108a0, 0x13})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:438 +0x1f25
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00b3c0e40, 0x0}, 0xc0065b1fc0, {0xc00bd7a210, 0x29}, 0xc00daaa6c0, {0x231b500, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0x433
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00072d800, {0x2339608, 0xc00005e018}, {0xc00b3c0e40, 0x23}, {0x231b500, 0xc00f83b380}, 0xc00457f5f0, {{0x0, 0x0, ...}, ...}, ...)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1889 +0x232c
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).operate(0xc00072d800, {0x2339608, 0xc00005e018})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:350 +0x16a8
github.com/argoproj/argo-workflows/v3/workflow/controller.(*WorkflowController).processNextItem(0xc000597000, {0x2339608, 0xc00005e018})
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/controller.go:756 +0x8ee
github.com/argoproj/argo-workflows/v3/workflow/controller.(*WorkflowController).runWorker(0x0)
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/controller.go:678 +0x9e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7f8744ad1120)
   /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x0, {0x22f6a00, 0xc00d759260}, 0x1, 0xc000526c60)
   /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x0, 0x0)
   /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(0x0, 0x0, 0x0)
   /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90 +0x25
created by github.com/argoproj/argo-workflows/v3/workflow/controller.(*WorkflowController).Run
   /go/src/github.com/argoproj/argo-workflows/workflow/controller/controller.go:294 +0x1a6c```
   
Is it possible to add more debug info? We absolutely have no idea why it's heppenening...

[sarabala1979]

@frimuchkov can you try v3.4-rc3?

[frimuchkov]

v3.4-rc3

I'll try :)

[frimuchkov]

@frimuchkov can you try v3.4-rc3?

@sarabala1979 During rolling out 3.4.0-rc3

time="2022-09-03T11:38:03.035Z" level=fatal msg="error unmarshaling JSON: while decoding JSON: json: unknown field \"archiveLogs\""

[sarabala1979]

@frimuchkov Are you getting the above error workflow controller log? can you provide more logs and your workflow yaml?

[frimuchkov]

@frimuchkov Are you getting the above error workflow controller log? can you provide more logs and your workflow yaml?

@sarabala1979 I'm getting it during rolling out new versions in argo-server and workflow controller Both servers can't start

Maybe I have to update some CRDs?

[frimuchkov]

@sarabala1979 deployed We had persistence.archiveLogs in argo-workflow-controller-configmap...

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. If this is a mentoring request, please provide an update here. Thank you for your contributions.

[stale[bot]]

This issue has been closed due to inactivity. Feel free to re-open if you still encounter this issue.