CI Failure (key symptom) in `OMBValidationTest.test_common_workload`

[vbotbuildovich]

https://buildkite.com/redpanda/vtools/builds/13732

Module: rptest.redpanda_cloud_tests.omb_validation_test
Class: OMBValidationTest
Method: test_common_workload

test_id:    OMBValidationTest.test_common_workload
status:     FAIL
run time:   116.063 seconds

CalledProcessError(1, ['tsh', 'ssh', '--proxy=proxy.tp.redpanda.com:443', '--auth=okta', '--identity=/tmp/machine-id/identity', 'redpanda@cp21mhnek221n77ef6k0-agent', 'kubectl', '--context=gke_devprod-cicd-infra_us-west1_redpanda-cp21mhnek221n77ef6k0', 'run', '--image docker.io/library/alpine', '--restart=Never', '--overrides=\'{"spec": {"nodeName": "gke-redpanda-cp21mhnek2-redpanda-a3e6-93906b52-fg26", "hostPID": true, "hostNetwork": true, "containers": [{"securityContext": {"privileged": true}, "image": "docker.io/library/alpine", "name": "nsenter", "stdin": true, "stdinOnce": true, "tty": true, "command": ["nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "bash", "-l"], "resources": {}, "volumeMounts": []}], "tolerations": [{"key": "CriticalAddonsOnly", "operator": "Exists"}, {"effect": "NoExecute", "operator": "Exists"}], "volumes": []}}\'', 'gke-redpanda-cp21mhnek2-redpanda-a3e6-93906b52-fg26-pshell'], '', 'Error from server (AlreadyExists): pods "gke-redpanda-cp21mhnek2-redpanda-a3e6-93906b52-fg26-pshell" already exists\n\x1b[31mERROR: \x1b[0mProcess exited with status 1\n\n')
Traceback (most recent call last):
  File "/opt/.ducktape-venv/lib/python3.10/site-packages/ducktape/tests/runner_client.py", line 177, in _do_run
    self.test = self.test_context.cls(self.test_context)
  File "/home/ubuntu/redpanda/tests/rptest/redpanda_cloud_tests/omb_validation_test.py", line 148, in __init__
    super().__init__(test_ctx, *args, **kwargs)
  File "/home/ubuntu/redpanda/tests/rptest/tests/redpanda_cloud_test.py", line 25, in __init__
    self.redpanda = make_redpanda_cloud_service(test_context)
  File "/home/ubuntu/redpanda/tests/rptest/services/redpanda.py", line 4928, in make_redpanda_cloud_service
    return RedpandaServiceCloud(context,
  File "/home/ubuntu/redpanda/tests/rptest/services/redpanda.py", line 1610, in __init__
    self.rebuild_pods_classes()
  File "/home/ubuntu/redpanda/tests/rptest/services/redpanda.py", line 1636, in rebuild_pods_classes
    self.pods = [
  File "/home/ubuntu/redpanda/tests/rptest/services/redpanda.py", line 1637, in <listcomp>
    CloudBroker(p, self.kubectl, self.logger)
  File "/home/ubuntu/redpanda/tests/rptest/services/cloud_broker.py", line 62, in __init__
    self.nodeshell.initialize_nodeshell()
  File "/home/ubuntu/redpanda/tests/rptest/clients/kubectl.py", line 469, in initialize_nodeshell
    _out = self.kubectl.cmd([
  File "/home/ubuntu/redpanda/tests/rptest/clients/kubectl.py", line 256, in cmd
    return self._ssh_cmd(cmd, capture=capture)
  File "/home/ubuntu/redpanda/tests/rptest/clients/kubectl.py", line 232, in _ssh_cmd
    return self._local_cmd(local_cmd)
  File "/home/ubuntu/redpanda/tests/rptest/clients/kubectl.py", line 215, in _local_cmd
    raise subprocess.CalledProcessError(process.returncode, cmd, s_out,
subprocess.CalledProcessError: Command '['tsh', 'ssh', '--proxy=proxy.tp.redpanda.com:443', '--auth=okta', '--identity=/tmp/machine-id/identity', 'redpanda@cp21mhnek221n77ef6k0-agent', 'kubectl', '--context=gke_devprod-cicd-infra_us-west1_redpanda-cp21mhnek221n77ef6k0', 'run', '--image docker.io/library/alpine', '--restart=Never', '--overrides=\'{"spec": {"nodeName": "gke-redpanda-cp21mhnek2-redpanda-a3e6-93906b52-fg26", "hostPID": true, "hostNetwork": true, "containers": [{"securityContext": {"privileged": true}, "image": "docker.io/library/alpine", "name": "nsenter", "stdin": true, "stdinOnce": true, "tty": true, "command": ["nsenter", "--target", "1", "--mount", "--uts", "--ipc", "--net", "--pid", "bash", "-l"], "resources": {}, "volumeMounts": []}], "tolerations": [{"key": "CriticalAddonsOnly", "operator": "Exists"}, {"effect": "NoExecute", "operator": "Exists"}], "volumes": []}}\'', 'gke-redpanda-cp21mhnek2-redpanda-a3e6-93906b52-fg26-pshell']' returned non-zero exit status 1.

JIRA Link: CORE-2977