for-linux
for-linux copied to clipboard
docker
Docker swarm not configuring default gateway on containers connected to overlay network
docker system info output on the manager:
Server:
Containers: 4
Running: 1
Paused: 0
Stopped: 3
Images: 7
Server Version: 20.10.17
Storage Driver: overlay2
Backing Filesystem: extfs
Supports d_type: true
Native Overlay Diff: true
userxattr: false
Logging Driver: json-file
Cgroup Driver: systemd
Cgroup Version: 2
Plugins:
Volume: local
Network: bridge host ipvlan macvlan null overlay
Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
Swarm: active
NodeID: lbwupro0h6kjs34p7gy2603m0
Is Manager: true
ClusterID: llsw4ilfu5xjedadjwlbtz4ff
Managers: 1
Nodes: 2
Default Address Pool: 10.0.0.0/8
SubnetSize: 24
Data Path Port: 4789
Orchestration:
Task History Retention Limit: 5
Raft:
Snapshot Interval: 10000
Number of Old Snapshots to Retain: 0
Heartbeat Tick: 1
Election Tick: 10
Dispatcher:
Heartbeat Period: 5 seconds
CA Configuration:
Expiry Duration: 3 months
Force Rotate: 0
Autolock Managers: false
Root Rotation In Progress: false
Node Address: X.X.X.X
Manager Addresses:
X.X.X.X:2377
Runtimes: io.containerd.runc.v2 io.containerd.runtime.v1.linux runc
Default Runtime: runc
Init Binary: docker-init
containerd version: 0197261a30bf81f1ee8e6a4dd2dea0ef95d67ccb
runc version: v1.1.3-0-g6724737
init version: de40ad0
Security Options:
apparmor
seccomp
Profile: default
cgroupns
Kernel Version: 5.10.0-17-amd64
Operating System: Debian GNU/Linux 11 (bullseye)
OSType: linux
Architecture: x86_64
CPUs: 6
Total Memory: 15.63GiB
Name: XXXXXXXXXXXXXXXXXX
ID: HUIY:DTNX:UDBR:2RNR:KFNE:J2WQ:J3RL:ZV3E:QLCP:T6WR:7QW6:M2DD
Docker Root Dir: /var/lib/docker
Debug Mode: false
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false
docker inspect traefik_traefik-network
[
{
"Name": "traefik_traefik-network",
"Id": "gre6shjq24dnp3lb20hajeyxl",
"Created": "2022-08-17T21:09:46.857818326+02:00",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.2.0/24",
"Gateway": "10.0.2.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee": {
"Name": "reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe",
"EndpointID": "532d9f9cc1d1127d4e03d5052054c7b96f8131c6b74e5d748bdf69323a7cacd5",
"MacAddress": "02:42:0a:00:02:04",
"IPv4Address": "10.0.2.4/24",
"IPv6Address": ""
},
"lb-traefik_traefik-network": {
"Name": "traefik_traefik-network-endpoint",
"EndpointID": "1bb5b3c913466a8de526bc5158cfc54f808fda5c01e10614660f4cdd799bafe9",
"MacAddress": "02:42:0a:00:02:05",
"IPv4Address": "10.0.2.5/24",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4098"
},
"Labels": {
"com.docker.stack.namespace": "reverse-proxy"
},
"Peers": [
{
"Name": "8d376eb566f8",
"IP": "X.X.X.X"
},
{
"Name": "372bc9b0c934",
"IP": "Y.Y.Y.Y"
}
]
}
]
docker container inspect reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe
[
{
"Id": "853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee",
"Created": "2022-08-17T19:09:43.013601794Z",
"Path": "/entrypoint.sh",
"Args": [
"traefik"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 4223,
"ExitCode": 0,
"Error": "",
"StartedAt": "2022-08-17T19:09:48.193302818Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:22c6901de2be6ce40b6c8b10426a5617c3a8715f78164f2408e94fe34e55110d",
"ResolvConfPath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/hostname",
"HostsPath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/hosts",
"LogPath": "/var/lib/docker/containers/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee/853176ebddfdfac0c298ee5ee4abae1366fe3a42c064f7384f3e4fe108a48dee-json.log",
"Name": "/reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "2",
"max-size": "50m",
"tag": "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}|{{index .ContainerLabels \"com.docker.compose.project\"}}|{{index .ContainerLabels \"com.docker.compose.service\"}}"
}
},
"NetworkMode": "default",
"PortBindings": {
"443/tcp": [
{
"HostIp": "",
"HostPort": "443"
}
],
"5432/tcp": [
{
"HostIp": "",
"HostPort": "5432"
}
],
"80/tcp": [
{
"HostIp": "",
"HostPort": "80"
}
]
},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "default",
"CpuShares": 0,
"Memory": 157286400,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 314572800,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": null,
"Ulimits": [],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"Mounts": [
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Target": "/var/run/docker.sock",
"ReadOnly": true
},
{
"Type": "volume",
"Source": "reverse-proxy_certificates",
"Target": "/acme",
"VolumeOptions": {
"Labels": {
"com.docker.stack.namespace": "reverse-proxy"
}
}
}
],
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c-init/diff:/var/lib/docker/overlay2/1782b79aa6d0329c2f4cb1882e2bea204c0118540b187b934277cad395b061c2/diff:/var/lib/docker/overlay2/aab0dadf085a30bd154795747ebd6ccfd0b2ec4f8c186b353ebda44dcf5df0e1/diff:/var/lib/docker/overlay2/e9973b879bada1914ca3f6593c9404db20f508c87fcf35721d0ec024dceb9ade/diff:/var/lib/docker/overlay2/6acecb8062f804cadc8962f8256b65b3f01ae6d81dcbe1caa7e381ec8f1709b3/diff",
"MergedDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c/merged",
"UpperDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c/diff",
"WorkDir": "/var/lib/docker/overlay2/c935a891ecfe09bd8b5b3b923747adacc52bd29e5f42d3f42a61790a7677647c/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Destination": "/var/run/docker.sock",
"Mode": "",
"RW": false,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "reverse-proxy_certificates",
"Source": "/var/lib/docker/volumes/reverse-proxy_certificates/_data",
"Destination": "/acme",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "853176ebddfd",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"443/tcp": {},
"5432/tcp": {},
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"traefik"
],
"Image": "traefik:v2.6@sha256:adf62c4c01290c29816f00b36a309df577771701e2feb2c9214fd3b6bd1e2894",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": [
"/entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"com.docker.stack.namespace": "reverse-proxy",
"com.docker.swarm.node.id": "lbwupro0h6kjs34p7gy2603m0",
"com.docker.swarm.service.id": "w5az89qy1m7mvtwkm6jrzox5o",
"com.docker.swarm.service.name": "reverse-proxy_reverse-proxy",
"com.docker.swarm.task": "",
"com.docker.swarm.task.id": "n8dxvft590laao30f7y1zspwe",
"com.docker.swarm.task.name": "reverse-proxy_reverse-proxy.1.n8dxvft590laao30f7y1zspwe",
"org.opencontainers.image.description": "A modern reverse-proxy",
"org.opencontainers.image.documentation": "https://docs.traefik.io",
"org.opencontainers.image.title": "Traefik",
"org.opencontainers.image.url": "https://traefik.io",
"org.opencontainers.image.vendor": "Traefik Labs",
"org.opencontainers.image.version": "v2.6.7"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "904f1f845cd275175af86830c77ef9f917bdf45953be14175b897b1e5289c58b",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"443/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "443"
},
{
"HostIp": "::",
"HostPort": "443"
}
],
"5432/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "5432"
},
{
"HostIp": "::",
"HostPort": "5432"
}
],
"80/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "80"
},
{
"HostIp": "::",
"HostPort": "80"
}
]
},
"SandboxKey": "/var/run/docker/netns/904f1f845cd2",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"traefik_traefik-network": {
"IPAMConfig": {
"IPv4Address": "10.0.2.4"
},
"Links": null,
"Aliases": [
"853176ebddfd"
],
"NetworkID": "gre6shjq24dnp3lb20hajeyxl",
"EndpointID": "532d9f9cc1d1127d4e03d5052054c7b96f8131c6b74e5d748bdf69323a7cacd5",
"Gateway": "",
"IPAddress": "10.0.2.4",
"IPPrefixLen": 24,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:0a:00:02:04",
"DriverOpts": null
}
}
}
}
]
As it's visible there is no Gateway assigned inside of the traefik container. Every other container regardless of node role also has none.
I already
- restarted docker
- restarted all nodes
- redloyed all containers
- recreating the networks
- Adding explicit IPTABLES rules to allow input of the ports required
The nodes are connected through an external IP address
Can anybody please tell me how to work around this issue?
We're not sure if this is a presentation or substantial issue, though it sounds like it's just a 'value is not in the API' issue; likewise Gateway seems to have always been flaky: https://github.com/moby/moby/issues/26799
@akerouanton mentioned that he will look at this (and testing with Stack, specifically) as part of his network test suite work in the future.
