Dragonfly2
Dragonfly2 copied to clipboard
No traces when pulling images from docker
Bug report:
I deployed dragonfly on AWS EKS and set 3 registries. Refer to the `values.yml' file below.
containerRuntime:
containerd:
enable: true
registries:
- "https://registry.hubble.jina.ai"
- "https://253352124568.dkr.ecr.us-east-2.amazonaws.com"
- "https://docker.io"
manager:
metrics:
enable: true
scheduler:
nodeSelector:
karpenter.sh/provisioner-name: system
metrics:
enable: true
seedPeer:
nodeSelector:
karpenter.sh/provisioner-name: system
metrics:
enable: true
persistence:
storageClass: "ebs-sc"
dfdaemon:
metrics:
enable: true
redis:
enable: true
global:
storageClass: "ebs-sc"
mysql:
enable: true
global:
storageClass: "ebs-sc"
jaeger:
enable: true
For each registry I have run gen-host.sh <host>
and setup /etc/containerd/config.toml
as shown here where certs.d
contains a dir for each host with the hosts.toml
file.
version = 2
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
When pulling images from docker no traces appear in jaeger
and no logs show when running kubectl -n dragonfly-system exec -it pod-name -- grep "peer task done" /var/log/dragonfly/daemon/core.log
. I am assuming the pod here should be the one that runs in the same node as the one where the image is pulled.
Here are the logs from dfdaemon
pod update-containerd
container
+ etcContainerd=/host/etc/containerd
+ '[[' -e /host/etc/containerd/config.toml ]]
+ echo containerd config found
+ cat /host/etc/containerd/config.toml
containerd config found
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
[grpc]
address = "/run/containerd/containerd.sock"
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
discard_unpacked_layers = true
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/pause:3.5"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d:/etc/docker/certs.d"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
+ registries='https://registry.hubble.jina.ai https://253352124568.dkr.ecr.us-east-2.amazonaws.com https://index.docker.io'
+ '[[' -n ]]
+ need_restart=0
+ grep 'version[^=]*=[^2]*2' /host/etc/containerd/config.toml
version = 2
+ cat /host/etc/containerd/config.toml
+ grep config_path
+ awk '{print $3}'
+ tr '"' ' '
config_path is enabled, add mirror in /etc/containerd/certs.d:/etc/docker/certs.d
+ config_path=/etc/containerd/certs.d:/etc/docker/certs.d
+ '[[' -z /etc/containerd/certs.d:/etc/docker/certs.d ]]
+ echo config_path is enabled, add mirror 'in' /etc/containerd/certs.d:/etc/docker/certs.d
+ cat /host/etc/containerd/config.toml
+ awk '{print $3}'
+ tr '"' ' '
+ grep config_path
+ tmp=/etc/containerd/certs.d:/etc/docker/certs.d
+ '[[' -z /etc/containerd/certs.d:/etc/docker/certs.d ]]
+ mkdir -p /host/etc/containerd/certs.d
+ echo https://registry.hubble.jina.ai
+ sed -e 's,http.*://,,'
+ sed 's,:.*,,'
+ domain=registry.hubble.jina.ai
+ mkdir -p /host/etc/containerd/certs.d/registry.hubble.jina.ai
registry https://registry.hubble.jina.ai found in config.toml, skip
+ '[[' -e /host/etc/containerd/certs.d/registry.hubble.jina.ai/hosts.toml ]]
+ echo 'registry https://registry.hubble.jina.ai found in config.toml, skip'
+ continue
+ sed -e 's,http.*://,,'
+ echo https://253352124568.dkr.ecr.us-east-2.amazonaws.com
+ sed 's,:.*,,'
+ domain=253352124568.dkr.ecr.us-east-2.amazonaws.com
+ mkdir -p /host/etc/containerd/certs.d/253352124568.dkr.ecr.us-east-2.amazonaws.com
+ '[[' -e /host/etc/containerd/certs.d/253352124568.dkr.ecr.us-east-2.amazonaws.com/hosts.toml ]]
+ echo 'registry https://253352124568.dkr.ecr.us-east-2.amazonaws.com found in config.toml, skip'
registry https://253352124568.dkr.ecr.us-east-2.amazonaws.com found in config.toml, skip
+ continue
+ echo https://index.docker.io
+ sed -e 's,http.*://,,'
+ sed 's,:.*,,'
+ domain=index.docker.io
+ mkdir -p /host/etc/containerd/certs.d/index.docker.io
registry https://index.docker.io found in config.toml, skip
+ '[[' -e /host/etc/containerd/certs.d/index.docker.io/hosts.toml ]]
+ echo 'registry https://index.docker.io found in config.toml, skip'
+ continue
+ '[[' 0 -gt 0 ]]
Expected behavior:
Docker images should be pulled through dragonfly
How to reproduce it:
Deploy dragonfly with docker registry in mirror mode.
Environment:
- Dragonfly version:
dfdaemon:v2.1.0
- OS:
linux
- Kernel (e.g.
uname -a
):5.10.186-179.751.amzn2.x86_64
- Others:
- containerd:
containerd://1.6.19
- kubelet:
v1.27.4
- containerd: