containerd
containerd copied to clipboard
Published
20 hours ago •
containerd
containerd
crictl pods - many pods NotReady after reboot, on crictl inspect i get "state": "SANDBOX_NOTREADY", "netNamespaceClosed": "true"
Description
I've opened an issue in K3s but I think it's also related to containrd. https://github.com/k3s-io/k3s/issues/11139 please see it if necessary.
crictl pods:
however all pods in kubectl are ok:
no empty files exist find /var/lib/cni/ -size 0 /var/lib/cni/networks/cbr0/lock
this is containerd logs: https://gist.github.com/UriZafrir/6844efe214678a087e3b5c080f0f916c
crictl inspectp gives: "state": "SANDBOX_NOTREADY": "netNamespaceClosed": true, pod has no ip.
this is true for all NotReady pods. crictl inspectp:
{
"status": {
"id": "cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
"metadata": {
"attempt": 0,
"name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"namespace": "kube-system",
"uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3"
},
"state": "SANDBOX_NOTREADY",
"createdAt": "2024-10-21T07:07:55.565102285Z",
"network": {
"additionalIps": [],
"ip": ""
},
"linux": {
"namespaces": {
"options": {
"ipc": "POD",
"network": "POD",
"pid": "CONTAINER",
"targetId": "",
"usernsOptions": null
}
}
},
"labels": {
"app": "svclb-xxx-bucket-service-9a49bf46",
"controller-revision-hash": "849b6c5d8",
"io.kubernetes.pod.name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"io.kubernetes.pod.namespace": "kube-system",
"io.kubernetes.pod.uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3",
"pod-template-generation": "1",
"svccontroller.k3s.cattle.io/svcname": "xxx-bucket-service",
"svccontroller.k3s.cattle.io/svcnamespace": "xxx"
},
"annotations": {
"kubernetes.io/config.seen": "2024-10-21T07:07:55.290397677Z",
"kubernetes.io/config.source": "api"
},
"runtimeHandler": ""
},
"info": {
"pid": 0,
"processStatus": "deleted",
"netNamespaceClosed": true,
"image": "docker.io/rancher/mirrored-pause:3.6",
"snapshotKey": "cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
"snapshotter": "overlayfs",
"runtimeHandler": "",
"runtimeType": "io.containerd.runc.v2",
"runtimeOptions": {
"systemd_cgroup": true
},
"config": {
"metadata": {
"name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3",
"namespace": "kube-system"
},
"hostname": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"log_directory": "/var/log/pods/kube-system_svclb-xxx-bucket-service-9a49bf46-jg2h6_85caaad1-0aa6-44e1-8a60-49fe06608ca3",
"dns_config": {
"servers": [
"10.43.0.10"
],
"searches": [
"kube-system.svc.cluster.local",
"svc.cluster.local",
"cluster.local"
],
"options": [
"ndots:5"
]
},
"port_mappings": [
{
"container_port": 5000,
"host_port": 5000
}
],
"labels": {
"app": "svclb-xxx-bucket-service-9a49bf46",
"controller-revision-hash": "849b6c5d8",
"io.kubernetes.pod.name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"io.kubernetes.pod.namespace": "kube-system",
"io.kubernetes.pod.uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3",
"pod-template-generation": "1",
"svccontroller.k3s.cattle.io/svcname": "xxx-bucket-service",
"svccontroller.k3s.cattle.io/svcnamespace": "xxx"
},
"annotations": {
"kubernetes.io/config.seen": "2024-10-21T07:07:55.290397677Z",
"kubernetes.io/config.source": "api"
},
"linux": {
"cgroup_parent": "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod85caaad1_0aa6_44e1_8a60_49fe06608ca3.slice",
"security_context": {
"namespace_options": {
"pid": 1
},
"seccomp": {}
},
"sysctls": {
"net.ipv4.ip_forward": "1",
"net.ipv4.ip_unprivileged_port_start": "0",
"net.ipv4.ping_group_range": "0 2147483647"
},
"overhead": {},
"resources": {
"cpu_period": 100000,
"cpu_shares": 2
}
}
},
"runtimeSpec": {
"ociVersion": "1.2.0",
"process": {
"user": {
"uid": 65535,
"gid": 65535,
"additionalGids": [
65535
]
},
"args": [
"/pause"
],
"env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"cwd": "/",
"capabilities": {
"bounding": [
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE"
],
"effective": [
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE"
],
"permitted": [
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FSETID",
"CAP_FOWNER",
"CAP_MKNOD",
"CAP_NET_RAW",
"CAP_SETGID",
"CAP_SETUID",
"CAP_SETFCAP",
"CAP_SETPCAP",
"CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT",
"CAP_KILL",
"CAP_AUDIT_WRITE"
]
},
"noNewPrivileges": true,
"oomScoreAdj": -998
},
"root": {
"path": "rootfs",
"readonly": true
},
"hostname": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"mounts": [
{
"destination": "/proc",
"type": "proc",
"source": "proc",
"options": [
"nosuid",
"noexec",
"nodev"
]
},
{
"destination": "/dev",
"type": "tmpfs",
"source": "tmpfs",
"options": [
"nosuid",
"strictatime",
"mode=755",
"size=65536k"
]
},
{
"destination": "/dev/pts",
"type": "devpts",
"source": "devpts",
"options": [
"nosuid",
"noexec",
"newinstance",
"ptmxmode=0666",
"mode=0620",
"gid=5"
]
},
{
"destination": "/dev/mqueue",
"type": "mqueue",
"source": "mqueue",
"options": [
"nosuid",
"noexec",
"nodev"
]
},
{
"destination": "/sys",
"type": "sysfs",
"source": "sysfs",
"options": [
"nosuid",
"noexec",
"nodev",
"ro"
]
},
{
"destination": "/dev/shm",
"type": "bind",
"source": "/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa/shm",
"options": [
"rbind",
"ro",
"nosuid",
"nodev",
"noexec"
]
},
{
"destination": "/etc/resolv.conf",
"type": "bind",
"source": "/var/lib/rancher/k3s/agent/containerd/io.containerd.grpc.v1.cri/sandboxes/cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa/resolv.conf",
"options": [
"rbind",
"ro",
"nosuid",
"nodev",
"noexec"
]
}
],
"annotations": {
"io.kubernetes.cri.container-type": "sandbox",
"io.kubernetes.cri.sandbox-cpu-period": "100000",
"io.kubernetes.cri.sandbox-cpu-quota": "0",
"io.kubernetes.cri.sandbox-cpu-shares": "2",
"io.kubernetes.cri.sandbox-id": "cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
"io.kubernetes.cri.sandbox-log-directory": "/var/log/pods/kube-system_svclb-xxx-bucket-service-9a49bf46-jg2h6_85caaad1-0aa6-44e1-8a60-49fe06608ca3",
"io.kubernetes.cri.sandbox-memory": "0",
"io.kubernetes.cri.sandbox-name": "svclb-xxx-bucket-service-9a49bf46-jg2h6",
"io.kubernetes.cri.sandbox-namespace": "kube-system",
"io.kubernetes.cri.sandbox-uid": "85caaad1-0aa6-44e1-8a60-49fe06608ca3"
},
"linux": {
"sysctl": {
"net.ipv4.ip_forward": "1",
"net.ipv4.ip_unprivileged_port_start": "0",
"net.ipv4.ping_group_range": "0 2147483647"
},
"resources": {
"devices": [
{
"allow": false,
"access": "rwm"
}
],
"cpu": {
"shares": 2
}
},
"cgroupsPath": "kubepods-besteffort-pod85caaad1_0aa6_44e1_8a60_49fe06608ca3.slice:cri-containerd:cf46d31391b99e9ca5f9ca29f4db027108e170f164c730079f8e72dc9abbacfa",
"namespaces": [
{
"type": "pid"
},
{
"type": "ipc"
},
{
"type": "uts"
},
{
"type": "mount"
},
{
"type": "network",
"path": "/var/run/netns/cni-769e973a-ba62-0965-fbfb-1daf0f978bb7"
}
],
"seccomp": {
"defaultAction": "SCMP_ACT_ERRNO",
"architectures": [
"SCMP_ARCH_ARM",
"SCMP_ARCH_AARCH64"
],
"syscalls": [
{
"names": [
"accept",
"accept4",
"access",
"adjtimex",
"alarm",
"bind",
"brk",
"cachestat",
"capget",
"capset",
"chdir",
"chmod",
"chown",
"chown32",
"clock_adjtime",
"clock_adjtime64",
"clock_getres",
"clock_getres_time64",
"clock_gettime",
"clock_gettime64",
"clock_nanosleep",
"clock_nanosleep_time64",
"close",
"close_range",
"connect",
"copy_file_range",
"creat",
"dup",
"dup2",
"dup3",
"epoll_create",
"epoll_create1",
"epoll_ctl",
"epoll_ctl_old",
"epoll_pwait",
"epoll_pwait2",
"epoll_wait",
"epoll_wait_old",
"eventfd",
"eventfd2",
"execve",
"execveat",
"exit",
"exit_group",
"faccessat",
"faccessat2",
"fadvise64",
"fadvise64_64",
"fallocate",
"fanotify_mark",
"fchdir",
"fchmod",
"fchmodat",
"fchmodat2",
"fchown",
"fchown32",
"fchownat",
"fcntl",
"fcntl64",
"fdatasync",
"fgetxattr",
"flistxattr",
"flock",
"fork",
"fremovexattr",
"fsetxattr",
"fstat",
"fstat64",
"fstatat64",
"fstatfs",
"fstatfs64",
"fsync",
"ftruncate",
"ftruncate64",
"futex",
"futex_requeue",
"futex_time64",
"futex_wait",
"futex_waitv",
"futex_wake",
"futimesat",
"getcpu",
"getcwd",
"getdents",
"getdents64",
"getegid",
"getegid32",
"geteuid",
"geteuid32",
"getgid",
"getgid32",
"getgroups",
"getgroups32",
"getitimer",
"getpeername",
"getpgid",
"getpgrp",
"getpid",
"getppid",
"getpriority",
"getrandom",
"getresgid",
"getresgid32",
"getresuid",
"getresuid32",
"getrlimit",
"get_robust_list",
"getrusage",
"getsid",
"getsockname",
"getsockopt",
"get_thread_area",
"gettid",
"gettimeofday",
"getuid",
"getuid32",
"getxattr",
"inotify_add_watch",
"inotify_init",
"inotify_init1",
"inotify_rm_watch",
"io_cancel",
"ioctl",
"io_destroy",
"io_getevents",
"io_pgetevents",
"io_pgetevents_time64",
"ioprio_get",
"ioprio_set",
"io_setup",
"io_submit",
"io_uring_enter",
"io_uring_register",
"io_uring_setup",
"ipc",
"kill",
"landlock_add_rule",
"landlock_create_ruleset",
"landlock_restrict_self",
"lchown",
"lchown32",
"lgetxattr",
"link",
"linkat",
"listen",
"listxattr",
"llistxattr",
"_llseek",
"lremovexattr",
"lseek",
"lsetxattr",
"lstat",
"lstat64",
"madvise",
"membarrier",
"memfd_create",
"memfd_secret",
"mincore",
"mkdir",
"mkdirat",
"mknod",
"mknodat",
"mlock",
"mlock2",
"mlockall",
"map_shadow_stack",
"mmap",
"mmap2",
"mprotect",
"mq_getsetattr",
"mq_notify",
"mq_open",
"mq_timedreceive",
"mq_timedreceive_time64",
"mq_timedsend",
"mq_timedsend_time64",
"mq_unlink",
"mremap",
"msgctl",
"msgget",
"msgrcv",
"msgsnd",
"msync",
"munlock",
"munlockall",
"munmap",
"name_to_handle_at",
"nanosleep",
"newfstatat",
"_newselect",
"open",
"openat",
"openat2",
"pause",
"pidfd_open",
"pidfd_send_signal",
"pipe",
"pipe2",
"pkey_alloc",
"pkey_free",
"pkey_mprotect",
"poll",
"ppoll",
"ppoll_time64",
"prctl",
"pread64",
"preadv",
"preadv2",
"prlimit64",
"process_mrelease",
"pselect6",
"pselect6_time64",
"pwrite64",
"pwritev",
"pwritev2",
"read",
"readahead",
"readlink",
"readlinkat",
"readv",
"recv",
"recvfrom",
"recvmmsg",
"recvmmsg_time64",
"recvmsg",
"remap_file_pages",
"removexattr",
"rename",
"renameat",
"renameat2",
"restart_syscall",
"rmdir",
"rseq",
"rt_sigaction",
"rt_sigpending",
"rt_sigprocmask",
"rt_sigqueueinfo",
"rt_sigreturn",
"rt_sigsuspend",
"rt_sigtimedwait",
"rt_sigtimedwait_time64",
"rt_tgsigqueueinfo",
"sched_getaffinity",
"sched_getattr",
"sched_getparam",
"sched_get_priority_max",
"sched_get_priority_min",
"sched_getscheduler",
"sched_rr_get_interval",
"sched_rr_get_interval_time64",
"sched_setaffinity",
"sched_setattr",
"sched_setparam",
"sched_setscheduler",
"sched_yield",
"seccomp",
"select",
"semctl",
"semget",
"semop",
"semtimedop",
"semtimedop_time64",
"send",
"sendfile",
"sendfile64",
"sendmmsg",
"sendmsg",
"sendto",
"setfsgid",
"setfsgid32",
"setfsuid",
"setfsuid32",
"setgid",
"setgid32",
"setgroups",
"setgroups32",
"setitimer",
"setpgid",
"setpriority",
"setregid",
"setregid32",
"setresgid",
"setresgid32",
"setresuid",
"setresuid32",
"setreuid",
"setreuid32",
"setrlimit",
"set_robust_list",
"setsid",
"setsockopt",
"set_thread_area",
"set_tid_address",
"setuid",
"setuid32",
"setxattr",
"shmat",
"shmctl",
"shmdt",
"shmget",
"shutdown",
"sigaltstack",
"signalfd",
"signalfd4",
"sigprocmask",
"sigreturn",
"socketcall",
"socketpair",
"splice",
"stat",
"stat64",
"statfs",
"statfs64",
"statx",
"symlink",
"symlinkat",
"sync",
"sync_file_range",
"syncfs",
"sysinfo",
"tee",
"tgkill",
"time",
"timer_create",
"timer_delete",
"timer_getoverrun",
"timer_gettime",
"timer_gettime64",
"timer_settime",
"timer_settime64",
"timerfd_create",
"timerfd_gettime",
"timerfd_gettime64",
"timerfd_settime",
"timerfd_settime64",
"times",
"tkill",
"truncate",
"truncate64",
"ugetrlimit",
"umask",
"uname",
"unlink",
"unlinkat",
"utime",
"utimensat",
"utimensat_time64",
"utimes",
"vfork",
"vmsplice",
"wait4",
"waitid",
"waitpid",
"write",
"writev"
],
"action": "SCMP_ACT_ALLOW"
},
{
"names": [
"socket"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 40,
"op": "SCMP_CMP_NE"
}
]
},
{
"names": [
"personality"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 0,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"personality"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 8,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"personality"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 131072,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"personality"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 131080,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"personality"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 4294967295,
"op": "SCMP_CMP_EQ"
}
]
},
{
"names": [
"process_vm_readv",
"process_vm_writev",
"ptrace"
],
"action": "SCMP_ACT_ALLOW"
},
{
"names": [
"arm_fadvise64_64",
"arm_sync_file_range",
"sync_file_range2",
"breakpoint",
"cacheflush",
"set_tls"
],
"action": "SCMP_ACT_ALLOW"
},
{
"names": [
"chroot"
],
"action": "SCMP_ACT_ALLOW"
},
{
"names": [
"clone"
],
"action": "SCMP_ACT_ALLOW",
"args": [
{
"index": 0,
"value": 2114060288,
"op": "SCMP_CMP_MASKED_EQ"
}
]
},
{
"names": [
"clone3"
],
"action": "SCMP_ACT_ERRNO",
"errnoRet": 38
}
]
},
"maskedPaths": [
"/proc/acpi",
"/proc/asound",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/sys/firmware",
"/sys/devices/virtual/powercap",
"/proc/scsi"
],
"readonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
}
},
"cniResult": {
"Interfaces": {
"cni0": {
"IPConfigs": null,
"Mac": "42:42:e0:e7:48:d7",
"Sandbox": ""
},
"eth0": {
"IPConfigs": [
{
"IP": "10.42.0.10",
"Gateway": "10.42.0.1"
}
],
"Mac": "8a:85:47:6a:8d:54",
"Sandbox": "/var/run/netns/cni-769e973a-ba62-0965-fbfb-1daf0f978bb7"
},
"lo": {
"IPConfigs": [
{
"IP": "127.0.0.1",
"Gateway": ""
},
{
"IP": "::1",
"Gateway": ""
}
],
"Mac": "00:00:00:00:00:00",
"Sandbox": "/var/run/netns/cni-769e973a-ba62-0965-fbfb-1daf0f978bb7"
},
"veth0270ece6": {
"IPConfigs": null,
"Mac": "5a:17:30:e1:db:0b",
"Sandbox": ""
}
},
"DNS": [
{},
{}
],
"Routes": [
{
"dst": "10.42.0.0/16"
},
{
"dst": "0.0.0.0/0",
"gw": "10.42.0.1"
}
]
}
}
}
Steps to reproduce the issue
- install k3s
- install metallb, two pods with helm:
helm install metallb metallb-chart/ --namespace metallb-system -f metallb-chart/values.yaml --debug --wait --timeout 10m
helm install xxxx helm-chart/ --namespace xxxx-f helm-chart/values-dev.yaml --debug --wait --timeout 60m
- reboot
Describe the results you received and expected
expected: crictl pods showing all pods ready received: crictl pods showing multiple pods not ready
What version of containerd are you using?
ctr github.com/k3s-io/containerd v1.7.15-k3s1
Any other relevant information
uname -a
Linux tegra-ubuntu 5.10.120-tegra #11 SMP PREEMPT Wed Nov 1 13:11:49 CST 2023 aarch64 aarch64 aarch64 GNU/Linux
crictl info
crictl info
{
"status": {
"conditions": [
{
"type": "RuntimeReady",
"status": true,
"reason": "",
"message": ""
},
{
"type": "NetworkReady",
"status": true,
"reason": "",
"message": ""
},
{
"type": "ContainerdHasNoDeprecationWarnings",
"status": false,
"reason": "ContainerdHasDeprecationWarnings",
"message": "{\"io.containerd.deprecation/cri-registry-configs\":\"The `configs` property of `[plugins.\\\"io.containerd.grpc.v1.cri\\\".registry]` is deprecated since containerd v1.5 and will be removed in containerd v2.0. Use `config_path` instead.\"}"
}
]
},
"cniconfig": {
"PluginDirs": [
"/var/lib/rancher/k3s/data/d85c4d741f84e813ef6cbfbf45702297c423d3f1a286057b3b1096c16c63bcc2/bin"
],
"PluginConfDir": "/var/lib/rancher/k3s/agent/etc/cni/net.d",
"PluginMaxConfNum": 1,
"Prefix": "eth",
"Networks": [
{
"Config": {
"Name": "cni-loopback",
"CNIVersion": "0.3.1",
"Plugins": [
{
"Network": {
"type": "loopback",
"ipam": {},
"dns": {}
},
"Source": "{\"type\":\"loopback\"}"
}
],
"Source": "{\n\"cniVersion\": \"0.3.1\",\n\"name\": \"cni-loopback\",\n\"plugins\": [{\n \"type\": \"loopback\"\n}]\n}"
},
"IFName": "lo"
},
{
"Config": {
"Name": "cbr0",
"CNIVersion": "1.0.0",
"Plugins": [
{
"Network": {
"type": "flannel",
"ipam": {},
"dns": {}
},
"Source": "{\"delegate\":{\"forceAddress\":true,\"hairpinMode\":true,\"isDefaultGateway\":true},\"type\":\"flannel\"}"
},
{
"Network": {
"type": "portmap",
"capabilities": {
"portMappings": true
},
"ipam": {},
"dns": {}
},
"Source": "{\"capabilities\":{\"portMappings\":true},\"type\":\"portmap\"}"
},
{
"Network": {
"type": "bandwidth",
"capabilities": {
"bandwidth": true
},
"ipam": {},
"dns": {}
},
"Source": "{\"capabilities\":{\"bandwidth\":true},\"type\":\"bandwidth\"}"
}
],
"Source": "{\n \"name\":\"cbr0\",\n \"cniVersion\":\"1.0.0\",\n \"plugins\":[\n {\n \"type\":\"flannel\",\n \"delegate\":{\n \"hairpinMode\":true,\n \"forceAddress\":true,\n \"isDefaultGateway\":true\n }\n },\n {\n \"type\":\"portmap\",\n \"capabilities\":{\n \"portMappings\":true\n }\n },\n {\n \"type\":\"bandwidth\",\n \"capabilities\":{\n \"bandwidth\":true\n }\n }\n ]\n}\n"
},
"IFName": "eth0"
}
]
},
"config": {
"containerd": {
"snapshotter": "overlayfs",
"defaultRuntimeName": "runc",
"defaultRuntime": {
"runtimeType": "",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": null,
"ContainerAnnotations": null,
"runtimeRoot": "",
"options": null,
"privileged_without_host_devices": false,
"privileged_without_host_devices_all_devices_allowed": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0,
"snapshotter": "",
"sandboxMode": ""
},
"untrustedWorkloadRuntime": {
"runtimeType": "",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": null,
"ContainerAnnotations": null,
"runtimeRoot": "",
"options": null,
"privileged_without_host_devices": false,
"privileged_without_host_devices_all_devices_allowed": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0,
"snapshotter": "",
"sandboxMode": ""
},
"runtimes": {
"runc": {
"runtimeType": "io.containerd.runc.v2",
"runtimePath": "",
"runtimeEngine": "",
"PodAnnotations": null,
"ContainerAnnotations": null,
"runtimeRoot": "",
"options": {
"SystemdCgroup": true
},
"privileged_without_host_devices": false,
"privileged_without_host_devices_all_devices_allowed": false,
"baseRuntimeSpec": "",
"cniConfDir": "",
"cniMaxConfNum": 0,
"snapshotter": "",
"sandboxMode": "podsandbox"
}
},
"noPivot": false,
"disableSnapshotAnnotations": true,
"discardUnpackedLayers": false,
"ignoreBlockIONotEnabledErrors": false,
"ignoreRdtNotEnabledErrors": false
},
"cni": {
"binDir": "/var/lib/rancher/k3s/data/d85c4d741f84e813ef6cbfbf45702297c423d3f1a286057b3b1096c16c63bcc2/bin",
"confDir": "/var/lib/rancher/k3s/agent/etc/cni/net.d",
"maxConfNum": 1,
"setupSerially": false,
"confTemplate": "",
"ipPref": ""
},
"registry": {
"configPath": "/var/lib/rancher/k3s/agent/etc/containerd/certs.d",
"mirrors": null,
"configs": {
"xxxx.azurecr.io": {
"auth": {
"username": "xxxx",
"password": "xxxx",
"auth": "",
"identitytoken": ""
},
"tls": null
}
},
"auths": null,
"headers": null
},
"imageDecryption": {
"keyModel": "node"
},
"disableTCPService": true,
"streamServerAddress": "127.0.0.1",
"streamServerPort": "10010",
"streamIdleTimeout": "4h0m0s",
"enableSelinux": false,
"selinuxCategoryRange": 1024,
"sandboxImage": "rancher/mirrored-pause:3.6",
"statsCollectPeriod": 10,
"systemdCgroup": false,
"enableTLSStreaming": false,
"x509KeyPairStreaming": {
"tlsCertFile": "",
"tlsKeyFile": ""
},
"maxContainerLogSize": 16384,
"disableCgroup": false,
"disableApparmor": false,
"restrictOOMScoreAdj": false,
"maxConcurrentDownloads": 3,
"disableProcMount": false,
"unsetSeccompProfile": "",
"tolerateMissingHugetlbController": true,
"disableHugetlbController": true,
"device_ownership_from_security_context": false,
"ignoreImageDefinedVolumes": false,
"netnsMountsUnderStateDir": false,
"enableUnprivilegedPorts": true,
"enableUnprivilegedICMP": true,
"enableCDI": false,
"cdiSpecDirs": [
"/etc/cdi",
"/var/run/cdi"
],
"imagePullProgressTimeout": "5m0s",
"drainExecSyncIOTimeout": "0s",
"imagePullWithSyncFs": false,
"ignoreDeprecationWarnings": null,
"containerdRootDir": "/var/lib/rancher/k3s/agent/containerd",
"containerdEndpoint": "/run/k3s/containerd/containerd.sock",
"rootDir": "/var/lib/rancher/k3s/agent/containerd/io.containerd.grpc.v1.cri",
"stateDir": "/run/k3s/containerd/io.containerd.grpc.v1.cri"
},
"golang": "go1.22.2",
"lastCNILoadStatus": "OK",
"lastCNILoadStatus.default": "OK"
}
Show configuration if it is related to CRI plugin.
cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml
# File generated by k3s. DO NOT EDIT. Use config.toml.tmpl instead.
version = 2
[plugins."io.containerd.internal.v1.opt"]
path = "/var/lib/rancher/k3s/agent/containerd"
[plugins."io.containerd.grpc.v1.cri"]
stream_server_address = "127.0.0.1"
stream_server_port = "10010"
enable_selinux = false
enable_unprivileged_ports = true
enable_unprivileged_icmp = true
sandbox_image = "rancher/mirrored-pause:3.6"
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "overlayfs"
disable_snapshot_annotations = true
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/var/lib/rancher/k3s/data/d85c4d741f84e813ef6cbfbf45702297c423d3f1a286057b3b1096c16c63bcc2/bin"
conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/var/lib/rancher/k3s/agent/etc/containerd/certs.d"
[plugins."io.containerd.grpc.v1.cri".registry.configs."xxxx.azurecr.io".auth]
username = "xxxx"
password = "xxxx"
Hi, @UriZafrir. I'm Dosu, and I'm helping the containerd team manage their backlog. I'm marking this issue as stale.
Issue Summary:
- Numerous pods are marked as "NotReady" after a system reboot.
crictlindicates network-related problems, whilekubectlshows pods as functioning correctly.- Discrepancy between
crictlandkubectlstatus reporting. - Suspected issue with containerd; logs provided for diagnosis.
- No additional comments or developments have been made.
Next Steps:
- Please confirm if this issue is still relevant to the latest version of the containerd repository by commenting here.
- If no updates are provided, the issue will be automatically closed in 7 days.
Thank you for your understanding and contribution!
![dosubot[bot] avatar](https://avatars.githubusercontent.com/in/324583?v=4 "Published by a pub.dev verified publisher"){kind=small}
Restarting the node will cause pods to be recreated. It is suspected that this occurs because the Kubernetes object has not yet updated its state, as the kubelet fails to propagate the pod status information during the restart. At this point, the information from crictl pods might be more accurate. After a full restart, it should reboot and return to normal.
