nydus
nydus copied to clipboard
Published
20 hours ago •
dragonflyoss
dragonflyoss
crictl create container failed
commandline:
root@peter-dev:/home/peter/Documents/tests/config/nydus# crictl pull localhost:5000/ubuntu-nydus:latest
DEBU[0000] get image connection
DEBU[0000] PullImageRequest: &PullImageRequest{Image:&ImageSpec{Image:localhost:5000/ubuntu-nydus:latest,Annotations:map[string]string{},},Auth:nil,SandboxConfig:nil,}
DEBU[0000] PullImageResponse: &PullImageResponse{ImageRef:sha256:2a355be4ad2c1712b40075c2c392d6d6b34d845d53c1e543ead1ddf829748387,}
Image is up to date for sha256:2a355be4ad2c1712b40075c2c392d6d6b34d845d53c1e543ead1ddf829748387
root@peter-dev:/home/peter/Documents/tests/config/nydus# pod=`sudo crictl runp nydus-sandbox.yaml`
DEBU[0000] get runtime connection
DEBU[0000] RunPodSandboxRequest: &RunPodSandboxRequest{Config:&PodSandboxConfig{Metadata:&PodSandboxMetadata{Name:nydus-sandbox-2,Uid:,Namespace:defaults,Attempt:1,},Hostname:,LogDirectory:/tmp,DnsConfig:nil,PortMappings:[]*PortMapping{},Labels:map[string]string{},Annotations:map[string]string{io.containerd.osfeature: nydus.remoteimage.v1,},Linux:&LinuxPodSandboxConfig{CgroupParent:,SecurityContext:&LinuxSandboxSecurityContext{NamespaceOptions:&NamespaceOption{Network:NODE,Pid:POD,Ipc:POD,TargetId:,},SelinuxOptions:nil,RunAsUser:nil,ReadonlyRootfs:false,SupplementalGroups:[],Privileged:false,SeccompProfilePath:,RunAsGroup:nil,Seccomp:nil,Apparmor:nil,},Sysctls:map[string]string{},Overhead:nil,Resources:nil,},Windows:nil,},RuntimeHandler:,}
DEBU[0004] RunPodSandboxResponse: 45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38
root@peter-dev:/home/peter/Documents/tests/config/nydus# $pod
45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38: command not found
root@peter-dev:/home/peter/Documents/tests/config/nydus# crictl pods
DEBU[0000] get runtime connection
DEBU[0000] ListPodSandboxRequest: &ListPodSandboxRequest{Filter:&PodSandboxFilter{Id:,State:nil,LabelSelector:map[string]string{},},}
DEBU[0000] ListPodSandboxResponse: [&PodSandbox{Id:45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38,Metadata:&PodSandboxMetadata{Name:nydus-sandbox-2,Uid:,Namespace:defaults,Attempt:1,},State:SANDBOX_READY,CreatedAt:1660559633942239838,Labels:map[string]string{},Annotations:map[string]string{io.containerd.osfeature: nydus.remoteimage.v1,},RuntimeHandler:,}]
POD ID CREATED STATE NAME NAMESPACE ATTEMPT RUNTIME
45b97dc94c515 18 seconds ago Ready nydus-sandbox-2 defaults 1 (default)
root@peter-dev:/home/peter/Documents/tests/config/nydus# container=`sudo crictl create $pod nydus-container.yaml nydus-sandbox.yaml`
DEBU[0000] get runtime connection
DEBU[0000] CreateContainerRequest: &CreateContainerRequest{PodSandboxId:45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38,Config:&ContainerConfig{Metadata:&ContainerMetadata{Name:nydus-container-1,Attempt:0,},Image:&ImageSpec{Image:localhost:5000/ubuntu-nydus:latest,Annotations:map[string]string{},},Command:[/bin/sleep],Args:[600],WorkingDir:,Envs:[]*KeyValue{},Mounts:[]*Mount{},Devices:[]*Device{},Labels:map[string]string{},Annotations:map[string]string{},LogPath:container.1.log,Stdin:false,StdinOnce:false,Tty:false,Linux:nil,Windows:nil,},SandboxConfig:&PodSandboxConfig{Metadata:&PodSandboxMetadata{Name:nydus-sandbox-2,Uid:,Namespace:defaults,Attempt:1,},Hostname:,LogDirectory:/tmp,DnsConfig:nil,PortMappings:[]*PortMapping{},Labels:map[string]string{},Annotations:map[string]string{io.containerd.osfeature: nydus.remoteimage.v1,},Linux:&LinuxPodSandboxConfig{CgroupParent:,SecurityContext:&LinuxSandboxSecurityContext{NamespaceOptions:&NamespaceOption{Network:NODE,Pid:POD,Ipc:POD,TargetId:,},SelinuxOptions:nil,RunAsUser:nil,ReadonlyRootfs:false,SupplementalGroups:[],Privileged:false,SeccompProfilePath:,RunAsGroup:nil,Seccomp:nil,Apparmor:nil,},Sysctls:map[string]string{},Overhead:nil,Resources:nil,},Windows:nil,},}
E0815 18:34:25.330796 4442 remote_runtime.go:421] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = DeadlineExceeded desc = context deadline exceeded" podSandboxID="45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38"
DEBU[0010] CreateContainerResponse:
FATA[0010] creating container: rpc error: code = DeadlineExceeded desc = context deadline exceeded
shared_daemon/nydusd.log :
[2022-08-15 18:30:38.887261 +08:00] INFO [app/src/lib.rs:68] Program Version: 2.1.0-alpha2, Git Commit: "10859e84d99d14bdbc9c53c82ca5963190c3f15b", Build Time: "Fri, 08 Jul 2022 06:26:32 +0000", Profile: "release", Rustc Version: "rustc 1.61.0 (fe5b13d68 2022-05-18)"
[2022-08-15 18:30:38.888222 +08:00] INFO [src/bin/nydusd/main.rs:513] Set rlimit-nofile to 1000000, maximum 1048576
[2022-08-15 18:30:38.889665 +08:00] INFO [/cargo/registry/src/github.com-1ecc6299db9ec823/fuse-backend-rs-0.9.0/src/transport/fusedev/linux_session.rs:332] mount source rafs dest /var/lib/containerd/io.containerd.snapshotter.v1.nydus/mnt with fstype fuse opts default_permissions,allow_other,fd=4,rootmode=40000,user_id=0,group_id=0 fd 4
[2022-08-15 18:30:38.891689 +08:00] INFO [src/bin/nydusd/daemon.rs:330] State machine(pid=3531): from Init to Ready, input [Mount], output [None]
[2022-08-15 18:30:38.891917 +08:00] INFO [src/bin/nydusd/daemon.rs:330] State machine(pid=3531): from Ready to Running, input [Start], output [Some(StartService)]
[2022-08-15 18:30:38.892062 +08:00] INFO [src/bin/nydusd/fusedev.rs:320] start 1 fuse servers
[2022-08-15 18:30:38.892409 +08:00] INFO [/cargo/registry/src/github.com-1ecc6299db9ec823/fuse-backend-rs-0.9.0/src/api/server/sync_io.rs:636] FUSE INIT major 7 minor 34
in_opts: ASYNC_READ | POSIX_LOCKS | ATOMIC_O_TRUNC | EXPORT_SUPPORT | BIG_WRITES | DONT_MASK | SPLICE_WRITE | SPLICE_MOVE | SPLICE_READ | FLOCK_LOCKS | HAS_IOCTL_DIR | AUTO_INVAL_DATA | DO_READDIRPLUS | READDIRPLUS_AUTO | ASYNC_DIO | WRITEBACK_CACHE | ZERO_MESSAGE_OPEN | PARALLEL_DIROPS | HANDLE_KILLPRIV | POSIX_ACL | ABORT_ERROR | MAX_PAGES | CACHE_SYMLINKS | ZERO_MESSAGE_OPENDIR | EXPLICIT_INVAL_DATA | HANDLE_KILLPRIV_V2
out_opts: ASYNC_READ | BIG_WRITES | HAS_IOCTL_DIR | AUTO_INVAL_DATA | DO_READDIRPLUS | READDIRPLUS_AUTO | ASYNC_DIO | WRITEBACK_CACHE | ZERO_MESSAGE_OPEN | PARALLEL_DIROPS | MAX_PAGES | CACHE_SYMLINKS | ZERO_MESSAGE_OPENDIR | EXPLICIT_INVAL_DATA
[2022-08-15 18:30:38.892737 +08:00] INFO [src/bin/nydusd/main.rs:659] Fuse daemon started!
[2022-08-15 18:30:38.893419 +08:00] INFO [src/bin/nydusd/api_server_glue.rs:406] HTTP API server running at /var/lib/containerd/io.containerd.snapshotter.v1.nydus/socket/shared_daemon/api.sock
[2022-08-15 18:30:38.893809 +08:00] INFO [api/src/http.rs:873] http server started
[2022-08-15 18:34:15.327131 +08:00] INFO [api/src/http.rs:778] <--- Post Uri { string: "/api/v1/mount?mountpoint=/3/fs" }
[2022-08-15 18:34:15.328613 +08:00] INFO [rafs/src/metadata/md_v5.rs:28] rafs superblock features: COMPRESS_LZ4_BLOCK | DIGESTER_BLAKE3 | EXPLICIT_UID_GID
[2022-08-15 18:34:15.329286 +08:00] INFO [storage/src/backend/connection.rs:182] backend config: ConnectionConfig { proxy: ProxyConfig { url: "", ping_url: "", fallback: false, check_interval: 5 }, skip_verify: false, timeout: 5, connect_timeout: 5, retry_limit: 2 }
[2022-08-15 18:34:15.340007 +08:00] INFO [src/bin/nydusd/fs_service.rs:202] Rafs imported
[2022-08-15 18:34:15.340044 +08:00] INFO [src/bin/nydusd/fs_service.rs:101] Rafs mounted at /3/fs
[2022-08-15 18:34:15.340126 +08:00] INFO [api/src/http.rs:783] ---> Post Status Code: NoContent, Elapse: Ok(12.997278ms), Body Size: 0
[2022-08-15 18:34:15.345202 +08:00] INFO [api/src/http.rs:778] <--- Get Uri { string: "/api/v1/daemon" }
[2022-08-15 18:34:15.345277 +08:00] INFO [api/src/http.rs:783] ---> Get Status Code: OK, Elapse: Ok(76.513µs), Body Size: 1072
[2022-08-15 18:34:15.347230 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 2
[2022-08-15 18:34:15.349981 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 2
[2022-08-15 18:34:15.352531 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 2
[2022-08-15 18:34:15.353048 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 2
[2022-08-15 18:34:15.353311 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 1
[2022-08-15 18:34:15.355903 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 1
[2022-08-15 18:34:15.360128 +08:00] ERROR [error/src/error.rs:21] Error:
Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n")))
at storage/src/cache/mod.rs:202
note: enable `RUST_BACKTRACE=1` env to display a backtrace
[2022-08-15 18:34:15.360847 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 1
[2022-08-15 18:34:15.361365 +08:00] WARN [storage/src/backend/mod.rs:86] Read from backend failed: Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n"))), retry count 1
[2022-08-15 18:34:15.361651 +08:00] ERROR [error/src/error.rs:21] Error:
Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n")))
at storage/src/cache/mod.rs:202
note: enable `RUST_BACKTRACE=1` env to display a backtrace
shared_daemon/nydus-snapshotter.log :
time="2022-08-15T18:30:38.874179190+08:00" level=info msg="gc goroutine start..."
time="2022-08-15T18:30:38.874386863+08:00" level=info msg="found 0 daemons running"
time="2022-08-15T18:30:38.875403225+08:00" level=warning msg="initialize the shared nydus daemon"
time="2022-08-15T18:30:38.876983172+08:00" level=info msg="start nydus daemon: /usr/local/bin/nydusd fuse --mountpoint /var/lib/containerd/io.containerd.snapshotter.v1.nydus/mnt --apisock /var/lib/containerd/io.containerd.snapshotter.v1.nydus/socket/shared_daemon/api.sock --log-level info --log-file /home/peter/Downloads/log/shared_daemon/nydusd.log"
time="2022-08-15T18:31:37.464264676+08:00" level=info msg="prepare snapshot with labels map[]" key="default/8/extract-460546904-eOys sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3" parent=
time="2022-08-15T18:31:37.464304074+08:00" level=info msg="prepare for container layer default/8/extract-460546904-eOys sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3" key="default/8/extract-460546904-eOys sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3" parent=
time="2022-08-15T18:31:37.464321109+08:00" level=info msg="id 1 is data layer, continue to check parent layer"
time="2022-08-15T18:31:37.521812203+08:00" level=info msg="remove snapshot with key default/8/extract-460546904-eOys sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3"
time="2022-08-15T18:31:37.523427902+08:00" level=info msg="cleanup: dirs=[/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/1]"
time="2022-08-15T18:31:37.523439781+08:00" level=info msg="cleanupSnapshotDirectory /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/1" dir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/1
time="2022-08-15T18:33:35.898574229+08:00" level=info msg="prepare snapshot with labels map[containerd.io/snapshot.ref:sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3 containerd.io/snapshot/cri.image-layers:sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3,sha256:b949f0b5c77f9e5968d37136b514f3f9208f4e2659919ea31f334f88551a29f0 containerd.io/snapshot/cri.image-ref:localhost:5000/ubuntu-nydus:latest containerd.io/snapshot/cri.layer-digest:sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3 containerd.io/snapshot/cri.manifest-digest:sha256:9dd8f32c64a1de07008fabc10cd7270da8ec0886058afb44e7deaeb60a82f39c containerd.io/snapshot/nydus-blob:true]" key="k8s.io/1/extract-893962400-_byt sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3" parent=
time="2022-08-15T18:33:35.898665335+08:00" level=info msg="nydus data layer, skip download and unpack k8s.io/1/extract-893962400-_byt sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3" key="k8s.io/1/extract-893962400-_byt sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3" parent=
time="2022-08-15T18:33:35.898697168+08:00" level=info msg="commit snapshot with key k8s.io/1/extract-893962400-_byt sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3"
time="2022-08-15T18:33:35.906796753+08:00" level=info msg="prepare snapshot with labels map[containerd.io/snapshot.ref:sha256:91823b59e5c7ef0502e855431c1a687d317111b19b008fd82f3251228d6de4b7 containerd.io/snapshot/cri.image-layers:sha256:b949f0b5c77f9e5968d37136b514f3f9208f4e2659919ea31f334f88551a29f0 containerd.io/snapshot/cri.image-ref:localhost:5000/ubuntu-nydus:latest containerd.io/snapshot/cri.layer-digest:sha256:b949f0b5c77f9e5968d37136b514f3f9208f4e2659919ea31f334f88551a29f0 containerd.io/snapshot/cri.manifest-digest:sha256:9dd8f32c64a1de07008fabc10cd7270da8ec0886058afb44e7deaeb60a82f39c containerd.io/snapshot/nydus-blob-ids:[\"af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3\"] containerd.io/snapshot/nydus-bootstrap:true containerd.io/snapshot/nydus-fs-version:5]" key="k8s.io/2/extract-904611811-R0OW sha256:91823b59e5c7ef0502e855431c1a687d317111b19b008fd82f3251228d6de4b7" parent="sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3"
time="2022-08-15T18:33:35.906864034+08:00" level=info msg="overlayfs mount options [workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/3/work upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/3/fs lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/2/fs]"
time="2022-08-15T18:33:35.924224982+08:00" level=info msg="commit snapshot with key k8s.io/2/extract-904611811-R0OW sha256:91823b59e5c7ef0502e855431c1a687d317111b19b008fd82f3251228d6de4b7"
time="2022-08-15T18:33:52.734512612+08:00" level=info msg="prepare snapshot with labels map[containerd.io/snapshot.ref:sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68 containerd.io/snapshot/cri.image-layers:sha256:fbe1a72f5dcd08ba4ca3ce3468c742786c1f6578c1f6bb401be1c4620d6ff705 containerd.io/snapshot/cri.image-ref:k8s.gcr.io/pause:3.6 containerd.io/snapshot/cri.layer-digest:sha256:fbe1a72f5dcd08ba4ca3ce3468c742786c1f6578c1f6bb401be1c4620d6ff705 containerd.io/snapshot/cri.manifest-digest:sha256:c2280d2f5f56cf9c9a01bb64b2db4651e35efd6d62a54dcfc12049fe6449c5e4]" key="k8s.io/4/extract-732247382-27Sz sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68" parent=
time="2022-08-15T18:33:53.926115390+08:00" level=info msg="commit snapshot with key k8s.io/4/extract-732247382-27Sz sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68"
time="2022-08-15T18:33:53.940014924+08:00" level=info msg="prepare snapshot with labels map[]" key=k8s.io/6/45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38 parent="k8s.io/5/sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68"
time="2022-08-15T18:33:53.940031150+08:00" level=info msg="prepare for container layer k8s.io/6/45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38" key=k8s.io/6/45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38 parent="k8s.io/5/sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68"
time="2022-08-15T18:33:53.940042685+08:00" level=info msg="id 5 is data layer, continue to check parent layer"
time="2022-08-15T18:33:53.940067731+08:00" level=info msg="id 4 is data layer, continue to check parent layer"
time="2022-08-15T18:33:53.940077814+08:00" level=info msg="overlayfs mount options [workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/5/work upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/5/fs lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/4/fs]"
time="2022-08-15T18:33:53.944419637+08:00" level=info msg="mount snapshot with key k8s.io/6/45b97dc94c515af56967343d86a4c4c09890322523b55d0bdf95b3859cef3d38"
time="2022-08-15T18:33:53.944445683+08:00" level=info msg="id 5 is data layer, continue to check parent layer"
time="2022-08-15T18:33:53.944455856+08:00" level=info msg="id 4 is data layer, continue to check parent layer"
time="2022-08-15T18:33:53.944468218+08:00" level=info msg="overlayfs mount options [workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/5/work upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/5/fs lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/4/fs]"
time="2022-08-15T18:34:15.323673162+08:00" level=info msg="prepare snapshot with labels map[]" key=k8s.io/7/eb44abfde26064457504e8c82c43070e0e2b00d7a4246897e2bf4121c5034a46 parent="k8s.io/3/sha256:91823b59e5c7ef0502e855431c1a687d317111b19b008fd82f3251228d6de4b7"
time="2022-08-15T18:34:15.323693230+08:00" level=info msg="prepare for container layer k8s.io/7/eb44abfde26064457504e8c82c43070e0e2b00d7a4246897e2bf4121c5034a46" key=k8s.io/7/eb44abfde26064457504e8c82c43070e0e2b00d7a4246897e2bf4121c5034a46 parent="k8s.io/3/sha256:91823b59e5c7ef0502e855431c1a687d317111b19b008fd82f3251228d6de4b7"
time="2022-08-15T18:34:15.323707078+08:00" level=info msg="id 6 is data layer, continue to check parent layer"
time="2022-08-15T18:34:15.323734323+08:00" level=info msg="found nydus meta layer id 3, parpare remote snapshot" key=k8s.io/7/eb44abfde26064457504e8c82c43070e0e2b00d7a4246897e2bf4121c5034a46 parent="k8s.io/3/sha256:91823b59e5c7ef0502e855431c1a687d317111b19b008fd82f3251228d6de4b7"
time="2022-08-15T18:34:15.323746204+08:00" level=info msg="prepare remote snapshot mountpoint /var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/3/fs"
time="2022-08-15T18:34:15.340232187+08:00" level=info msg="image localhost:5000/ubuntu-nydus:latest with blob caches [af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3]"
time="2022-08-15T18:34:15.342589794+08:00" level=info msg="remote mount options [workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/6/work upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/6/fs lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/mnt/3/fs]"
time="2022-08-15T18:34:15.345067393+08:00" level=info msg="mount snapshot with key k8s.io/7/eb44abfde26064457504e8c82c43070e0e2b00d7a4246897e2bf4121c5034a46"
time="2022-08-15T18:34:15.345105378+08:00" level=info msg="id 6 is data layer, continue to check parent layer"
time="2022-08-15T18:34:15.345382070+08:00" level=info msg="remote mount options [workdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/6/work upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/snapshots/6/fs lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.nydus/mnt/3/fs]"
time="2022-08-15T18:35:21.313501469+08:00" level=info msg="caught signal {}: shutting down"
time="2022-08-15T18:35:21.313720135+08:00" level=info msg="nydus snapshotter exited"
containerd: ###
Aug 15 18:39:12 peter-dev containerd[2795]: time="2022-08-15T18:39:12.991076067+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:39:23 peter-dev containerd[2795]: time="2022-08-15T18:39:23.167642842+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:39:33 peter-dev containerd[2795]: time="2022-08-15T18:39:33.835191236+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:39:43 peter-dev containerd[2795]: time="2022-08-15T18:39:43.194444951+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:39:53 peter-dev containerd[2795]: time="2022-08-15T18:39:53.335255328+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:40:03 peter-dev containerd[2795]: time="2022-08-15T18:40:03.672753593+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:40:13 peter-dev containerd[2795]: time="2022-08-15T18:40:13.278945977+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:40:23 peter-dev containerd[2795]: time="2022-08-15T18:40:23.241915980+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:40:32 peter-dev containerd[2795]: time="2022-08-15T18:40:32.846073897+08:00" level=error msg="Failed to sync snapshot>
Aug 15 18:40:42 peter-dev containerd[2795]: time="2022-08-15T18:40:42.884853410+08:00" level=error msg="Failed to sync snapshot>
nydus version : v2.1.0-alpha.5 nydus snapshotter version : v0.3.0-alpha.5 containerd github.com/containerd/containerd v1.6.6 10c12954828e7c7c9b6e0ea9b0c02b01407d3ae1
@Zhuchengyu04 Could you share the nydud config.json file?
nydusd-config.json
{
"device": {
"backend": {
"type": "registry",
"config": {
"scheme": "http",
"skip_verify": false,
"timeout": 5,
"connect_timeout": 5,
"retry_limit": 2
}
},
"cache": {
"type": "blobcache",
"config": {
"work_dir": "cache"
}
}
},
"mode": "direct",
"digest_validate": false,
"iostats_files": false,
"enable_xattr": true,
"fs_prefetch": {
"enable": true,
"threads_count": 4
}
}
ERROR [error/src/error.rs:21] Error:
Registry(Request(ErrorWithMsg("<!doctype html>\n<html>\n<head>\n<meta http-equiv=\"refresh\" content=\"0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL\"/>\n</head>\n<body>\n</body>\n</html>\n")))
at storage/src/cache/mod.rs:202
note: enable `RUST_BACKTRACE=1` env to display a backtrace
Looks like some problem was encountered when requesting this http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3.
What happens when try to curl http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3 directly?
What happens when try to
curl http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3directly?
root@peter-dev:/home/peter/Downloads/accelerated-container-image# curl http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3
<!doctype html>
<html>
<head>
<meta http-equiv="refresh" content="0;url=http://7rx80271.ibosscloud.com/ibreports/ibp/bp.html?bu=http://localhost:5000/v2/ubuntu-nydus/blobs/sha256:af6d37f391f92eff5d182127d8507fa15e05ba1f600975f84cab325dc8c428c3&bc=Failed+to+retrieve+requested+URL.&ip=10.239.159.24&er=ERR_CONNECT_FAIL"/>
</head>
<body>
</body>
</html>
@Zhuchengyu04 Please check why the request to http://localhost:5000 is redirected to http://7rx80271.ibosscloud.com.
@Zhuchengyu04 Please check why the request to
http://localhost:5000is redirected tohttp://7rx80271.ibosscloud.com.
Sorry, I have no idea about it.
@Zhuchengyu04 Please check why the request to
http://localhost:5000is redirected tohttp://7rx80271.ibosscloud.com.Sorry, I have no idea about it.
I think you could use 127.0.0.1?
@Zhuchengyu04 Please check why the request to
http://localhost:5000is redirected tohttp://7rx80271.ibosscloud.com.
I find the address is intercepted by my proxy.
[2022-08-16 23:03:22.039008 +08:00] DEBUG [/cargo/registry/src/github.com-1ecc6299db9ec823/reqwest-0.11.11/src/connect.rs:433] proxy(http://xxxxxx.xxx.xx:xx) intercepts 'http://localhost:5000/'
@Zhuchengyu04 Please check why the request to
http://localhost:5000is redirected tohttp://7rx80271.ibosscloud.com.Sorry, I have no idea about it.
I think you could use 127.0.0.1?
Sorry, I tried but it's not working. and now there has a new problem.
# crictl create cd75fc14b7ae1 nydus-container.yaml nydus-sandbox.yaml
DEBU[0000] get runtime connection
DEBU[0000] CreateContainerRequest: &CreateContainerRequest{PodSandboxId:cd75fc14b7ae1,Config:&ContainerConfig{Metadata:&ContainerMetadata{Name:nydus-container,Attempt:0,},Image:&ImageSpec{Image:localhost:5000/ubuntu-nydus:latest,Annotations:map[string]string{},},Command:[/bin/sleep],Args:[600],WorkingDir:,Envs:[]*KeyValue{},Mounts:[]*Mount{},Devices:[]*Device{},Labels:map[string]string{},Annotations:map[string]string{},LogPath:container.1.log,Stdin:false,StdinOnce:false,Tty:false,Linux:nil,Windows:nil,},SandboxConfig:&PodSandboxConfig{Metadata:&PodSandboxMetadata{Name:nydus-sandbox,Uid:hdishd83djaidwnduwk28bcsd,Namespace:default,Attempt:1,},Hostname:,LogDirectory:/tmp,DnsConfig:nil,PortMappings:[]*PortMapping{},Labels:map[string]string{},Annotations:map[string]string{io.containerd.osfeature: nydus.remoteimage.v1,},Linux:&LinuxPodSandboxConfig{CgroupParent:,SecurityContext:&LinuxSandboxSecurityContext{NamespaceOptions:&NamespaceOption{Network:NODE,Pid:POD,Ipc:POD,TargetId:,},SelinuxOptions:nil,RunAsUser:nil,ReadonlyRootfs:false,SupplementalGroups:[],Privileged:false,SeccompProfilePath:,RunAsGroup:nil,Seccomp:nil,Apparmor:nil,},Sysctls:map[string]string{},Overhead:nil,Resources:nil,},Windows:nil,},}
E0816 23:17:05.218577 259802 remote_runtime.go:421] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to create containerd container: daemon 0IVrnx0fSmOEXZZxOLlYPQ is not ready: &{ {2.1.0-alpha2 10859e84d99d14bdbc9c53c82ca5963190c3f15b Fri, 08 Jul 2022 06:26:32 +0000 release rustc 1.61.0 (fe5b13d68 2022-05-18)} INIT}: unknown" podSandboxID="cd75fc14b7ae1"
DEBU[0000] CreateContainerResponse:
FATA[0000] creating container: rpc error: code = Unknown desc = failed to create containerd container: daemon 0IVrnx0fSmOEXZZxOLlYPQ is not ready: &{ {2.1.0-alpha2 10859e84d99d14bdbc9c53c82ca5963190c3f15b Fri, 08 Jul 2022 06:26:32 +0000 release rustc 1.61.0 (fe5b13d68 2022-05-18)} INIT}: unknown
@Zhuchengyu04 Please check why the request to
http://localhost:5000is redirected tohttp://7rx80271.ibosscloud.com.Sorry, I have no idea about it.
I think you could use 127.0.0.1?
Sorry, I tried but it's not working. and now there has a new problem.
# crictl create cd75fc14b7ae1 nydus-container.yaml nydus-sandbox.yaml DEBU[0000] get runtime connection DEBU[0000] CreateContainerRequest: &CreateContainerRequest{PodSandboxId:cd75fc14b7ae1,Config:&ContainerConfig{Metadata:&ContainerMetadata{Name:nydus-container,Attempt:0,},Image:&ImageSpec{Image:localhost:5000/ubuntu-nydus:latest,Annotations:map[string]string{},},Command:[/bin/sleep],Args:[600],WorkingDir:,Envs:[]*KeyValue{},Mounts:[]*Mount{},Devices:[]*Device{},Labels:map[string]string{},Annotations:map[string]string{},LogPath:container.1.log,Stdin:false,StdinOnce:false,Tty:false,Linux:nil,Windows:nil,},SandboxConfig:&PodSandboxConfig{Metadata:&PodSandboxMetadata{Name:nydus-sandbox,Uid:hdishd83djaidwnduwk28bcsd,Namespace:default,Attempt:1,},Hostname:,LogDirectory:/tmp,DnsConfig:nil,PortMappings:[]*PortMapping{},Labels:map[string]string{},Annotations:map[string]string{io.containerd.osfeature: nydus.remoteimage.v1,},Linux:&LinuxPodSandboxConfig{CgroupParent:,SecurityContext:&LinuxSandboxSecurityContext{NamespaceOptions:&NamespaceOption{Network:NODE,Pid:POD,Ipc:POD,TargetId:,},SelinuxOptions:nil,RunAsUser:nil,ReadonlyRootfs:false,SupplementalGroups:[],Privileged:false,SeccompProfilePath:,RunAsGroup:nil,Seccomp:nil,Apparmor:nil,},Sysctls:map[string]string{},Overhead:nil,Resources:nil,},Windows:nil,},} E0816 23:17:05.218577 259802 remote_runtime.go:421] "CreateContainer in sandbox from runtime service failed" err="rpc error: code = Unknown desc = failed to create containerd container: daemon 0IVrnx0fSmOEXZZxOLlYPQ is not ready: &{ {2.1.0-alpha2 10859e84d99d14bdbc9c53c82ca5963190c3f15b Fri, 08 Jul 2022 06:26:32 +0000 release rustc 1.61.0 (fe5b13d68 2022-05-18)} INIT}: unknown" podSandboxID="cd75fc14b7ae1" DEBU[0000] CreateContainerResponse: FATA[0000] creating container: rpc error: code = Unknown desc = failed to create containerd container: daemon 0IVrnx0fSmOEXZZxOLlYPQ is not ready: &{ {2.1.0-alpha2 10859e84d99d14bdbc9c53c82ca5963190c3f15b Fri, 08 Jul 2022 06:26:32 +0000 release rustc 1.61.0 (fe5b13d68 2022-05-18)} INIT}: unknown
Is there any clue in nydusd.log.
