Parent node not initialized

[frimuchkov]

Hi!

After updating to 3.4.0-rc3 we got rig of "null pointer exceptions", but started to see errors "parent node not initialized".

time="2022-09-12T05:45:28.379Z" level=error msg="Recovered from panic" namespace=default r="parent node  not initialized" stack="goroutine 266 [running]:
runtime/debug.Stack()
  /usr/local/go/src/runtime/debug/stack.go:24 +0x65
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).operate.func2()
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:190 +0xbc
panic({0x1d5fd80, 0xc000aa0700})
  /usr/local/go/src/runtime/panic.go:844 +0x258
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).addChildNode(0xc00041cbb0, {0x0, 0x0}, {0xc007a47b80, 0x122})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:2900 +0x35e
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask.func1({0xc007a47b80, 0x122})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:429 +0x175
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, 0xc0077413b0, {0xc002dfaba1, 0x10})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:453 +0x1daa
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, 0xc0077413b0, {0xc0097c1120, 0x10})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:438 +0x1f05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc007a60c60, 0x111}, 0xc008146cc0, {0xc009bcd4a0, 0x23}, 0xc00a1fb8c0, {0x24ab5c8, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0xf05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc007a60c60, 0x111}, {0x24ab5c8, 0xc00a1883c0}, 0x0?, {{0xc00a482000, 0x15, ...}, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1891 +0x22ec
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, 0xc007741260, {0xc009b83780, 0x33})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1368
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc009f88a80, 0xdd}, 0xc008146b00, {0xc009bcce10, 0x23}, 0xc00a1fa000, {0x24ac9a8, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0xf05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc009f88a80, 0xdd}, {0x24ac9a8, 0xc007a56540}, 0x6d2f223a22687461?, {{0xc007a211e0, 0x3, ...}, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1891 +0x22ec
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeStepGroup(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc005167200, 0x1, 0x4}, {0xc003c0d0e0, 0x5c}, 0xc00a970700)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/steps.go:247 +0x605
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeSteps(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc003c0cf00, 0x59}, 0xc0049afc00, {0xc0069fdd10, 0x23}, 0xc00776c480, {0x24ab5c8, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/steps.go:95 +0xcc5
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc003c0cf00, 0x59}, {0x24ab5c8, 0xc00774a960}, 0x0?, {{0x0, 0x0, ...}, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1885 +0x242c
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, 0xc007740690, {0xc0074d6c60, 0x1a})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1368
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc0172df300, 0x3e}, 0xc0049af9c0, {0xc0069fdb00, 0x23}, 0xc007747440, {0x24ab5c8, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0xf05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc0172df300, 0x3e}, {0x24ab5c8, 0xc00774a3c0}, 0x0?, {{0x0, 0x0, ...}, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1891 +0x22ec
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, 0xc0077404d0, {0xc0077480e1, 0x20})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:513 +0x1368
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAGTask(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, 0xc0077404d0, {0xc004942b58, 0x13})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:438 +0x1f05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeDAG(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc0031f45a0, 0x1d}, 0xc0049af740, {0xc0069fd740, 0x23}, 0xc007746480, {0x24ac9a8, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/dag.go:244 +0xf05
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).executeTemplate(0xc00041cbb0, {0x24be5a0, 0xc00005e018}, {0xc0031f45a0, 0x1d}, {0x24ac9a8, 0xc004bacf00}, 0x1cd2fc0?, {{0x0, 0x0, ...}, ...}, ...)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:1891 +0x22ec
github.com/argoproj/argo-workflows/v3/workflow/controller.(*wfOperationCtx).operate(0xc00041cbb0, {0x24be5a0, 0xc00005e018})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/operator.go:344 +0x16ed
github.com/argoproj/argo-workflows/v3/workflow/controller.(*WorkflowController).processNextItem(0xc0002f5900, {0x24be5a0, 0xc00005e018})
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/controller.go:752 +0x759
github.com/argoproj/argo-workflows/v3/workflow/controller.(*WorkflowController).runWorker(0xc0005876a0?)
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/controller.go:674 +0x9e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x0?)
  /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x0?, {0x249c8a0, 0xc0040bca80}, 0x1, 0xc000415140)
  /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000e28130?, 0x3b9aca00, 0x0, 0xf0?, 0x16c2800?)
  /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(0xc000a6fb40?, 0xa0180c001?, 0xc0005877a0?)
  /go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90 +0x25
created by github.com/argoproj/argo-workflows/v3/workflow/controller.(*WorkflowController).Run
  /go/src/github.com/argoproj/argo-workflows/workflow/controller/controller.go:308 +0x19c5
" workflow=project-overland-com-f653c682

[sarabala1979]

@frimuchkov Can you provide more information? reproducible workflow, controller log,

[frimuchkov]

@sarabala1979 but I provided logs above :) We rolled back to 3.3.9 and started to see "nill pointer", then we added --loglevel warn and error disappeared, looks like it was a problem with disk and a lot of logs Will try to upgrade to 3.4.0-rc3 back

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. If this is a mentoring request, please provide an update here. Thank you for your contributions.

[stale[bot]]

This issue has been closed due to inactivity. Feel free to re-open if you still encounter this issue.