kubeblocks
kubeblocks copied to clipboard
apecloud
[BUG] etcd cluster create with clientService error
Describe the bug A clear and concise description of what the bug is.
kbcli version
Kubernetes: v1.28.3-vke.17
KubeBlocks: 1.0.0-beta.33
kbcli: 1.0.0-beta.15
helm get notes -n kb-system kb-addon-etcd
Release Information:
Commit ID: "62f551fe1961d33210ea1c2234c290619a42f509"
Commit Time: "2025-03-03 16:52:29 +0800"
Release Branch: "v1.0.0-beta.33"
Release Time: "2025-03-12 15:08:52 +0800"
Enterprise: "false"
probe event failed {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2-etcd-0.41cdfc11","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd-0.41cdfc11", "reconcileID": "0288c18f-bc8c-40de-92a7-40bad1cfcd4c", "event": {"name":"etcd-cluster2-etcd-0.41cdfc11","namespace":"default"}, "message": "exit code: 1, stderr: {"level":"warn","ts":"2025-03-14T07:39:09.523455Z","logger":"etcd-client","caller":"[email protected]/retry_interceptor.go:63","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00031e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""}\nFailed to get the status of endpoint 127.0.0.1:2379 (context deadline exceeded)\netcdctl command failed\nbad role, please check!\n: failed"}
To Reproduce Steps to reproduce the behavior:
- create etcd cluster
helm template etcd-cluster addons-cluster/etcd
---
# Source: etcd-cluster/templates/cluster.yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
name: etcd-cluster
namespace: default
labels:
helm.sh/chart: etcd-cluster-1.0.0-alpha.0
app.kubernetes.io/version: "3.5.15"
app.kubernetes.io/instance: etcd-cluster
spec:
terminationPolicy: Delete
componentSpecs:
- name: etcd
componentDef: etcd
serviceVersion: 3.5.15
tls: false
replicas: 3
volumeClaimTemplates:
- name: data # ref clusterDefinition components.containers.volumeMounts.name
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
resources:
limits:
cpu: "0.5"
memory: "0.5Gi"
requests:
cpu: "0.5"
memory: "0.5Gi"
disableExporter: false
➜ ~ kubectl get cluster etcd-cluster
NAME CLUSTER-DEFINITION TERMINATION-POLICY STATUS AGE
etcd-cluster Delete Running 2m11s
➜ ~ kbcli cluster list-instances etcd-cluster
NAME NAMESPACE CLUSTER COMPONENT STATUS ROLE ACCESSMODE AZ CPU(REQUEST/LIMIT) MEMORY(REQUEST/LIMIT) STORAGE NODE CREATED-TIME
etcd-cluster-etcd-0 default etcd-cluster etcd Running leader cn-shanghai-a 500m / 500m 512Mi / 512Mi data:10Gi 172.16.0.26/172.16.0.26 Mar 14,2025 15:32 UTC+0800
etcd-cluster-etcd-1 default etcd-cluster etcd Running follower cn-shanghai-a 500m / 500m 512Mi / 512Mi data:10Gi 172.16.0.6/172.16.0.6 Mar 14,2025 15:32 UTC+0800
etcd-cluster-etcd-2 default etcd-cluster etcd Running follower cn-shanghai-a 500m / 500m 512Mi / 512Mi data:10Gi 172.16.0.26/172.16.0.26 Mar 14,2025 15:32 UTC+0800
➜ ~
- create etcd cluster with clientService
helm template etcd-cluster2 addons-cluster/etcd --set clientService.name=etcd
---
# Source: etcd-cluster/templates/cluster.yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
name: etcd-cluster2
namespace: default
labels:
helm.sh/chart: etcd-cluster-1.0.0-alpha.0
app.kubernetes.io/version: "3.5.15"
app.kubernetes.io/instance: etcd-cluster2
spec:
terminationPolicy: Delete
componentSpecs:
- name: etcd
componentDef: etcd
serviceVersion: 3.5.15
tls: false
replicas: 3
volumeClaimTemplates:
- name: data # ref clusterDefinition components.containers.volumeMounts.name
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
resources:
limits:
cpu: "0.5"
memory: "0.5Gi"
requests:
cpu: "0.5"
memory: "0.5Gi"
disableExporter: false
services:
- name: etcd
serviceName: etcd
spec:
type: ClusterIP
ports:
- port: 2379
targetPort: 2379
componentSelector: etcd
roleSelector: leader
- See error
➜ ~ kubectl get cluster etcd-cluster2
NAME CLUSTER-DEFINITION TERMINATION-POLICY STATUS AGE
etcd-cluster2 Delete Failed 46s
➜ ~ kubectl get pod -l app.kubernetes.io/instance=etcd-cluster2
NAME READY STATUS RESTARTS AGE
etcd-cluster2-etcd-0 1/2 CrashLoopBackOff 2 (27s ago) 67s
logs error pod
kubectl logs etcd-cluster2-etcd-0 --previous
Defaulted container "etcd" out of: etcd, kbagent, inject-bash (init), init-kbagent (init), kbagent-worker (init)
[2025-03-14 07:37:00] start to rebuild etcd configuration...
[2025-03-14 07:37:00] LoadBalancer mode detected. Adapting pod FQDN to balance IP.
[2025-03-14 07:37:00] Failed to get my peer endpoint from PEER_FQDNS:etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local,etcd-cluster2-etcd-1.etcd-cluster2-etcd-headless.default.svc.cluster.local,etcd-cluster2-etcd-2.etcd-cluster2-etcd-headless.default.svc.cluster.local when loadBalancer mode is enabled, use default pod FQDN to advertise.
[2025-03-14 07:37:00] Updated etcd.conf:
# https://github.com/etcd-io/etcd/blob/main/etcd.conf.yml.sample
# using this config file will ignore ALL command-line flag and environment variables.
# Human-readable name for this member.
name: etcd-cluster2-etcd-0
# Path to the data directory.
data-dir: /var/run/etcd/default.etcd
# Path to the dedicated wal directory.
wal-dir:
# Number of committed transactions to trigger a snapshot to disk.
snapshot-count: 10000
# Time (in milliseconds) of a heartbeat interval.
heartbeat-interval: 100
# Time (in milliseconds) for an election to timeout.
election-timeout: 1000
# Raise alarms when backend size exceeds the given quota. 0 means use the
# default quota.
quota-backend-bytes: 0
# List of comma separated URLs to listen on for peer traffic.
listen-peer-urls: http://0.0.0.0:2380
# List of comma separated URLs to listen on for client traffic.
listen-client-urls: http://0.0.0.0:2379
# Maximum number of snapshot files to retain (0 is unlimited).
max-snapshots: 5
# Maximum number of wal files to retain (0 is unlimited).
max-wals: 5
# Comma-separated white list of origins for CORS (cross-origin resource sharing).
cors:
# List of this member's peer URLs to advertise to the rest of the cluster.
# The URLs needed to be a comma-separated list.
initial-advertise-peer-urls: http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2380
# List of this member's client URLs to advertise to the public.
# The URLs needed to be a comma-separated list.
advertise-client-urls: http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2379
# Discovery URL used to bootstrap the cluster.
discovery:
# Valid values include 'exit', 'proxy'
discovery-fallback: 'proxy'
# HTTP proxy to use for traffic to discovery service.
discovery-proxy:
# DNS domain used to bootstrap initial cluster.
discovery-srv:
# Comma separated string of initial cluster configuration for bootstrapping.
# Example: initial-cluster: "infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380"
initial-cluster: etcd-cluster2-etcd=http://etcd-cluster2-etcd:2380
# Initial cluster token for the etcd cluster during bootstrap.
initial-cluster-token: 'etcd-cluster'
# Initial cluster state ('new' or 'existing').
initial-cluster-state: 'new'
# Reject reconfiguration requests that would cause quorum loss.
strict-reconfig-check: true
# Enable runtime profiling data via HTTP server
enable-pprof: true
# Valid values include 'on', 'readonly', 'off'
proxy: 'off'
# Time (in milliseconds) an endpoint will be held in a failed state.
proxy-failure-wait: 5000
# Time (in milliseconds) of the endpoints refresh interval.
proxy-refresh-interval: 30000
# Time (in milliseconds) for a dial to timeout.
proxy-dial-timeout: 1000
# Time (in milliseconds) for a write to timeout.
proxy-write-timeout: 5000
# Time (in milliseconds) for a read to timeout.
proxy-read-timeout: 0
# The validity period of the self-signed certificate, the unit is year.
self-signed-cert-validity: 1
# Enable info-level logging for etcd.
log-level: info
logger: zap
# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
log-outputs: [stderr]
# Force to create a new one member cluster.
force-new-cluster: false
auto-compaction-mode: periodic
auto-compaction-retention: "1"
# Limit etcd to a specific set of tls cipher suites
cipher-suites: [
TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
]
# Limit etcd to specific TLS protocol versions
tls-min-version: 'TLS1.2'
tls-max-version: 'TLS1.3'
# Enable to check data corruption before serving any client/peer traffic.
experimental-initial-corrupt-check: true
[2025-03-14 07:37:00] ---
[2025-03-14 07:37:00] Rebuilt etcd configuration successfully.
[2025-03-14 07:37:00] Starting etcd with updated configuration...
{"level":"warn","ts":"2025-03-14T07:37:00.456774Z","caller":"embed/config.go:687","msg":"Running http and grpc server on single port. This is not recommended for production."}
{"level":"info","ts":"2025-03-14T07:37:00.456964Z","caller":"etcdmain/config.go:353","msg":"loaded server configuration, other configuration command line flags and environment variables will be ignored if provided","path":"/var/run/etcd/etcd.conf"}
{"level":"info","ts":"2025-03-14T07:37:00.456976Z","caller":"etcdmain/etcd.go:73","msg":"Running: ","args":["etcd","--config-file","/var/run/etcd/etcd.conf"]}
{"level":"info","ts":"2025-03-14T07:37:00.457011Z","caller":"etcdmain/etcd.go:116","msg":"server has been already initialized","data-dir":"/var/run/etcd/default.etcd","dir-type":"member"}
{"level":"warn","ts":"2025-03-14T07:37:00.457023Z","caller":"embed/config.go:687","msg":"Running http and grpc server on single port. This is not recommended for production."}
{"level":"info","ts":"2025-03-14T07:37:00.457030Z","caller":"embed/etcd.go:128","msg":"configuring peer listeners","listen-peer-urls":["http://0.0.0.0:2380"]}
{"level":"info","ts":"2025-03-14T07:37:00.457123Z","caller":"embed/etcd.go:136","msg":"configuring client listeners","listen-client-urls":["http://0.0.0.0:2379"]}
{"level":"info","ts":"2025-03-14T07:37:00.457136Z","caller":"embed/etcd.go:620","msg":"pprof is enabled","path":"/debug/pprof"}
{"level":"info","ts":"2025-03-14T07:37:00.457196Z","caller":"embed/etcd.go:310","msg":"starting an etcd server","etcd-version":"3.5.15","git-sha":"9a5533382","go-version":"go1.21.12","go-os":"linux","go-arch":"amd64","max-cpu-set":4,"max-cpu-available":4,"member-initialized":false,"name":"etcd-cluster2-etcd-0","data-dir":"/var/run/etcd/default.etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/run/etcd/default.etcd/member","force-new-cluster":false,"heartbeat-interval":"100ms","election-timeout":"1s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2380"],"listen-peer-urls":["http://0.0.0.0:2380"],"advertise-client-urls":["http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2379"],"listen-client-urls":["http://0.0.0.0:2379"],"listen-metrics-urls":[],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"etcd-cluster2-etcd=http://etcd-cluster2-etcd:2380","initial-cluster-state":"new","initial-cluster-token":"etcd-cluster","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"periodic","auto-compaction-retention":"1h0m0s","auto-compaction-interval":"1h0m0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"}
{"level":"info","ts":"2025-03-14T07:37:00.457788Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/run/etcd/default.etcd/member/snap/db","took":"421.106µs"}
{"level":"info","ts":"2025-03-14T07:37:00.458294Z","caller":"embed/etcd.go:377","msg":"closing etcd server","name":"etcd-cluster2-etcd-0","data-dir":"/var/run/etcd/default.etcd","advertise-peer-urls":["http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2380"],"advertise-client-urls":["http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2379"]}
{"level":"info","ts":"2025-03-14T07:37:00.458317Z","caller":"embed/etcd.go:379","msg":"closed etcd server","name":"etcd-cluster2-etcd-0","data-dir":"/var/run/etcd/default.etcd","advertise-peer-urls":["http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2380"],"advertise-client-urls":["http://etcd-cluster2-etcd-0.etcd-cluster2-etcd-headless.default.svc.cluster.local:2379"]}
{"level":"fatal","ts":"2025-03-14T07:37:00.458330Z","caller":"etcdmain/etcd.go:204","msg":"discovery failed","error":"couldn't find local name \"etcd-cluster2-etcd-0\" in the initial cluster configuration","stacktrace":"go.etcd.io/etcd/server/v3/etcdmain.startEtcdOrProxyV2\n\tgo.etcd.io/etcd/server/v3/etcdmain/etcd.go:204\ngo.etcd.io/etcd/server/v3/etcdmain.Main\n\tgo.etcd.io/etcd/server/v3/etcdmain/main.go:40\nmain.main\n\tgo.etcd.io/etcd/server/v3/main.go:31\nruntime.main\n\truntime/proc.go:267"}
logs kubeblocks pod
kubectl logs -n kb-system kubeblocks-847db6ff6b-5rbd2|grep etcd-cluster2
Defaulted container "manager" out of: manager, tools (init)
2025-03-14T07:35:49.409Z INFO update cluster status after applying resources, generation: 1 {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "7e7bbbdc-c400-42fe-bda1-3e28e52d63cc", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:35:49.427Z INFO reconcile object *v1.Component with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "c3416450-7b8b-4cad-a020-a3aca663dcd3", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.441Z INFO ComponentParameterReconciler ComponentParameter created {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "7fa850f6-9d26-4d81-b11e-1de122c2d4b3", "Namespace": "default", "Parameter": "etcd-cluster2-etcd"}
2025-03-14T07:35:49.452Z INFO ComponentParameterReconciler.etcd-config ComponentParameters cr is creating: {etcd-config map[componentResource:[123 34 108 105 109 105 116 115 34 58 123 34 99 112 117 34 58 34 53 48 48 109 34 44 34 109 101 109 111 114 121 34 58 34 53 49 50 77 105 34 125 44 34 114 101 113 117 101 115 116 115 34 58 123 34 99 112 117 34 58 34 53 48 48 109 34 44 34 109 101 109 111 114 121 34 58 34 53 49 50 77 105 34 125 125] replicas:[51]] 0xc002a00ff0 <nil> map[]} {"controller": "componentparameter", "controllerGroup": "parameters.kubeblocks.io", "controllerKind": "ComponentParameter", "ComponentParameter": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "22ba462b-ad0b-487d-9c7c-1e2b7bb8a85e", "Namespace": "default", "ComponentParameter": "etcd-cluster2-etcd"}
2025-03-14T07:35:49.461Z INFO update cluster status after applying resources, generation: 2 {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "ddf42a0a-3e80-4103-8675-e635304469de", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:35:49.488Z INFO reconcile object *v1.ConfigMap with action CREATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "cc3186a9-0666-4f78-aba0-5c5ef4b21ebf", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.494Z INFO reconcile object *v1.ConfigMap with action CREATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "cc3186a9-0666-4f78-aba0-5c5ef4b21ebf", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.504Z INFO reconcile object *v1.ConfigMap with action CREATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "cc3186a9-0666-4f78-aba0-5c5ef4b21ebf", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.513Z INFO reconcile object *v1.InstanceSet with action CREATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "cc3186a9-0666-4f78-aba0-5c5ef4b21ebf", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.518Z INFO execute error: Operation cannot be fulfilled on components.apps.kubeblocks.io "etcd-cluster2-etcd": the object has been modified; please apply your changes to the latest version and try again {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "cc3186a9-0666-4f78-aba0-5c5ef4b21ebf", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.543Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "c4dea18f-eaf4-4e7b-827f-b39ef479bfd6", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.567Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "d331211d-b78a-403e-a29c-0d7eb30654a3", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.581Z INFO execute error: Operation cannot be fulfilled on instancesets.workloads.kubeblocks.io "etcd-cluster2-etcd": the object has been modified; please apply your changes to the latest version and try again {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "d331211d-b78a-403e-a29c-0d7eb30654a3", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.584Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "6dc4d69a-2fb3-4999-9909-2e80189f5d70", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.599Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "9dc0f49e-d0a6-40cf-a4c8-b4845a600cd7", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.618Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "bb831a75-db47-4df6-9c3b-71a8ffba0fab", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.625Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "bd10fa12-9e41-4fa1-84e3-cd4c026259ea", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.636Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "fac07459-1f91-4b91-a1b5-9daba9b50a0c", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.638Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "bd10fa12-9e41-4fa1-84e3-cd4c026259ea", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.644Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "bd10fa12-9e41-4fa1-84e3-cd4c026259ea", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.652Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "3e04de2a-60ef-4138-a8fd-d21c3d91b93d", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.672Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "f50004a2-d17b-477b-b95f-ca0ad8888ad2", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.710Z INFO cluster component phase transition: -> Creating (cluster component etcd is Creating) {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "aaaa5b66-acb3-4bfa-b81c-00df9b038a2b", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:35:49.839Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "fd2f7280-ded2-4b39-869a-ea3f99cc4153", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.851Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "fd2f7280-ded2-4b39-869a-ea3f99cc4153", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.856Z INFO execute error: Operation cannot be fulfilled on components.apps.kubeblocks.io "etcd-cluster2-etcd": the object has been modified; please apply your changes to the latest version and try again {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "fd2f7280-ded2-4b39-869a-ea3f99cc4153", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:49.861Z INFO cluster component phase transition: -> Creating (cluster component etcd is Creating) {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "07a8e630-d603-4afe-988e-3e906a8254e9", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:35:49.870Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "9ca71390-daaf-4e1e-ae3f-8129cebc5d0c", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:49.870Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "9ca71390-daaf-4e1e-ae3f-8129cebc5d0c", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:49.879Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "0c8d95d4-25b9-489d-bb2c-f09990f9b584", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:49.879Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "0c8d95d4-25b9-489d-bb2c-f09990f9b584", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:49.892Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "5771d675-d56f-4de2-bf37-2eb3b0c67923", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:49.892Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "5771d675-d56f-4de2-bf37-2eb3b0c67923", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:49.916Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "0b4e3233-484e-45aa-9f22-5a0557ddd281", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:49.916Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "0b4e3233-484e-45aa-9f22-5a0557ddd281", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:50.060Z INFO cluster component phase transition: -> Creating (cluster component etcd is Creating) {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "d8bac830-f993-45fc-8de4-c655151bd72f", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:35:50.069Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "db059d1c-5c9f-47cd-b68a-750b3f28dd61", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:50.069Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "db059d1c-5c9f-47cd-b68a-750b3f28dd61", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:50.079Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "89a0dac3-ff31-44da-a79e-624b3a73a209", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:50.079Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "89a0dac3-ff31-44da-a79e-624b3a73a209", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:50.086Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "89f02e03-90b8-48c3-b040-97c294dd71a9", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:50.093Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "86945664-3c42-4f79-ab09-1073160a5976", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:50.093Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "86945664-3c42-4f79-ab09-1073160a5976", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:50.097Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "89f02e03-90b8-48c3-b040-97c294dd71a9", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:50.103Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "89f02e03-90b8-48c3-b040-97c294dd71a9", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:50.117Z INFO eventHandledError {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "fa9c8049-4bfb-4068-be64-568482326cf8", "event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}}
2025-03-14T07:35:50.117Z ERROR Reconciler error {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2.182c9b2778564d34","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2.182c9b2778564d34", "reconcileID": "fa9c8049-4bfb-4068-be64-568482326cf8", "error": "Too many requests: limit reached on type User for key system:serviceaccount:kb-system:kubeblocks"}
2025-03-14T07:35:50.339Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "03377fd5-576f-42d5-a297-d6b22c298da3", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:50.351Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "03377fd5-576f-42d5-a297-d6b22c298da3", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:50.357Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "03377fd5-576f-42d5-a297-d6b22c298da3", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.440Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "18bb95fc-ec4e-4995-a3de-82c9e8ff300d", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.452Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "18bb95fc-ec4e-4995-a3de-82c9e8ff300d", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.458Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "18bb95fc-ec4e-4995-a3de-82c9e8ff300d", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.499Z INFO status conditions, creating: true, its running: false, has failure: false, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "135bc40b-ce57-4d92-ae51-ed4e84abd75b", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.510Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "135bc40b-ce57-4d92-ae51-ed4e84abd75b", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.516Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "135bc40b-ce57-4d92-ae51-ed4e84abd75b", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.590Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "86d9200c-1028-4a68-860c-dd8e45e04409", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:35:52.610Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "f1ebc754-19b0-42fc-a382-885d1aa30f7a", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:02.916Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "40ddb1e4-9be6-4551-b480-8d67134fbd09", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:05.922Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "3120057a-cd29-44b2-bc57-57d553b604d3", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:06.924Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "49d62828-6f0f-430e-a79a-48afabcac2d9", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:07.926Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "148b3016-013c-4e46-a80c-b19b38932ce5", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:09.935Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "917c0518-b1af-4b20-abbe-073eddce9519", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:09.964Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "f1dd9cc6-f97d-4f86-9df5-16d3c2d2c35f", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.006Z INFO status conditions, creating: true, its running: false, has failure: true, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "57754a65-59ce-4368-93c0-830745836ac5", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.025Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "57754a65-59ce-4368-93c0-830745836ac5", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.033Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "57754a65-59ce-4368-93c0-830745836ac5", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.040Z INFO cluster component phase transition: -> Failed (cluster component etcd is Failed) {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "6ebaea48-09e2-4152-80ca-0aa84e20f8e7", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:36:10.070Z INFO cluster component phase transition: -> Failed (cluster component etcd is Failed) {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "a85f8bff-c637-4154-8e99-31031dae27c1", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:36:10.084Z INFO status conditions, creating: false, its running: false, has failure: true, updating: false {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "1346945f-6c76-404a-be5e-da4739fa36cb", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.135Z INFO reconcile object *v1.InstanceSet with action UPDATE OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "1346945f-6c76-404a-be5e-da4739fa36cb", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.141Z INFO reconcile object *v1.Component with action STATUS OK {"controller": "component", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Component", "Component": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "1346945f-6c76-404a-be5e-da4739fa36cb", "component": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:10.146Z INFO cluster component phase transition: -> Failed (cluster component etcd is Failed) {"controller": "cluster", "controllerGroup": "apps.kubeblocks.io", "controllerKind": "Cluster", "Cluster": {"name":"etcd-cluster2","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2", "reconcileID": "133debfb-2e7e-48e3-b339-13fe0d94807e", "cluster": {"name":"etcd-cluster2","namespace":"default"}}
2025-03-14T07:36:10.937Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "a3bf193a-f78b-4ad8-9f45-39b4837e4b8f", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:11.317Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "da6297ec-56df-48e9-b095-f596e13dd7f1", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:11.939Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "5db0db1a-7c2b-4ba6-8915-f7a27f6323fa", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:29.975Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "74524fa3-5977-4de5-bc93-ae16f2b865fe", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:36:44.838Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "16198e53-42e4-431d-8181-b6757b219841", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:37:01.028Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "643591e8-d631-4d60-9638-1992b9f2c7ea", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:37:09.549Z INFO probe event failed {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2-etcd-0.1ccf0867","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd-0.1ccf0867", "reconcileID": "09f0060d-9806-4144-9241-e831888306a9", "event": {"name":"etcd-cluster2-etcd-0.1ccf0867","namespace":"default"}, "message": "exit code: 1, stderr: {\"level\":\"warn\",\"ts\":\"2025-03-14T07:37:09.529634Z\",\"logger\":\"etcd-client\",\"caller\":\"[email protected]/retry_interceptor.go:63\",\"msg\":\"retrying of unary invoker failed\",\"target\":\"etcd-endpoints://0xc0003e8000/127.0.0.1:2379\",\"attempt\":0,\"error\":\"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \\\"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\\\"\"}\nFailed to get the status of endpoint 127.0.0.1:2379 (context deadline exceeded)\netcdctl command failed\nbad role, please check!\n: failed"}
2025-03-14T07:37:14.838Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "afb37b2a-1a48-46b0-8ae8-a92794238005", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:37:43.094Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "967c42f2-dda9-4fb0-82bd-14d1a2a824f0", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:37:53.838Z INFO InstanceSet default/etcd-cluster2-etcd instances are not aligned {"controller": "instanceset", "controllerGroup": "workloads.kubeblocks.io", "controllerKind": "InstanceSet", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd", "reconcileID": "53b39412-98ca-4058-8e0e-46bd0b3d9f95", "InstanceSet": {"name":"etcd-cluster2-etcd","namespace":"default"}}
2025-03-14T07:38:09.543Z INFO probe event failed {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2-etcd-0.a35822b9","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd-0.a35822b9", "reconcileID": "fdc5fe72-3668-45bf-8e78-83109eb160d8", "event": {"name":"etcd-cluster2-etcd-0.a35822b9","namespace":"default"}, "message": "exit code: 1, stderr: {\"level\":\"warn\",\"ts\":\"2025-03-14T07:38:09.531464Z\",\"logger\":\"etcd-client\",\"caller\":\"[email protected]/retry_interceptor.go:63\",\"msg\":\"retrying of unary invoker failed\",\"target\":\"etcd-endpoints://0xc00037a000/127.0.0.1:2379\",\"attempt\":0,\"error\":\"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \\\"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\\\"\"}\nFailed to get the status of endpoint 127.0.0.1:2379 (context deadline exceeded)\netcdctl command failed\nbad role, please check!\n: failed"}
2025-03-14T07:39:09.537Z INFO probe event failed {"controller": "event", "controllerGroup": "", "controllerKind": "Event", "Event": {"name":"etcd-cluster2-etcd-0.41cdfc11","namespace":"default"}, "namespace": "default", "name": "etcd-cluster2-etcd-0.41cdfc11", "reconcileID": "0288c18f-bc8c-40de-92a7-40bad1cfcd4c", "event": {"name":"etcd-cluster2-etcd-0.41cdfc11","namespace":"default"}, "message": "exit code: 1, stderr: {\"level\":\"warn\",\"ts\":\"2025-03-14T07:39:09.523455Z\",\"logger\":\"etcd-client\",\"caller\":\"[email protected]/retry_interceptor.go:63\",\"msg\":\"retrying of unary invoker failed\",\"target\":\"etcd-endpoints://0xc00031e000/127.0.0.1:2379\",\"attempt\":0,\"error\":\"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \\\"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\\\"\"}\nFailed to get the status of endpoint 127.0.0.1:2379 (context deadline exceeded)\netcdctl command failed\nbad role, please check!\n: failed"}
Expected behavior A clear and concise description of what you expected to happen.
Screenshots If applicable, add screenshots to help explain your problem.
Desktop (please complete the following information):
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
Additional context Add any other context about the problem here.
It it wired. If I set client Name to 'etcdclient', the cluster starts as expected. Will check the details.
helm template etcd-cluster2 addons-cluster/etcd --set clientService.name=etcdclient
---
# Source: etcd-cluster/templates/cluster.yaml
apiVersion: apps.kubeblocks.io/v1
kind: Cluster
metadata:
name: etcd-cluster2
namespace: default
labels:
helm.sh/chart: etcd-cluster-1.0.0-alpha.0
app.kubernetes.io/version: "3.5.15"
app.kubernetes.io/instance: etcd-cluster2
spec:
terminationPolicy: Delete
componentSpecs:
- name: etcd
componentDef: etcd
serviceVersion: 3.5.15
tls: false
replicas: 3
volumeClaimTemplates:
- name: data # ref clusterDefinition components.containers.volumeMounts.name
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
resources:
limits:
cpu: "0.5"
memory: "0.5Gi"
requests:
cpu: "0.5"
memory: "0.5Gi"
disableExporter: false
services:
- name: etcdclient
serviceName: etcdclient
spec:
type: ClusterIP
ports:
- port: 2379
targetPort: 2379
componentSelector: etcd
roleSelector: leader
This issue has been marked as stale because it has been open for 30 days with no activity
![github-actions[bot] avatar](https://avatars.githubusercontent.com/in/15368?v=4 "Published by a pub.dev verified publisher"){kind=small}
When create cluster service was named 'etcd', it match the default headless svc name and then pollute init_peers, so I hard code the svc name in cluster.
