for-linux
for-linux copied to clipboard
docker
Docker ignores DNS setting (still using embedded DNS) if custom network used
- [x] This is a bug report
- [ ] This is a feature request
- [ ] I searched existing issues before opening this one
Expected behavior
The /etc/resolv.conf file should contain setting from HostConfig.Dns
Actual behavior
# docker inspect ad00015b1e67
[
{
"Id": "ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c",
"Created": "2018-05-31T10:11:01.631588125Z",
"Path": "/usr/local/bin/entrypoint.sh",
"Args": [
"/opt/python/latest/bin/gunicorn",
"ras.wsgi",
"-b",
"0.0.0.0:80",
"-w",
"2"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 809954,
"ExitCode": 0,
"Error": "",
"StartedAt": "2018-05-31T10:11:27.945132303Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:3d0c2dbd95f3fe57ab8a1a637e2c09b11da80c442eb0304f4b5443286c6c955e",
"ResolvConfPath": "/var/lib/docker/containers/ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c/hostname",
"HostsPath": "/var/lib/docker/containers/ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c/hosts",
"LogPath": "/var/lib/docker/containers/ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c/ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c-json.log",
"Name": "/mesos-d020f9c3-252a-41cc-8498-20c8a7e3fba2",
"RestartCount": 0,
"Driver": "devicemapper",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/var/lib/mesos/slaves/c9fc1b80-454a-4ada-be8b-4d6e770791d6-S1/frameworks/762936cb-22f7-437f-ae8b-eb355e206474-0000/executors/ads80-p10-tst08.eb010f99-64ba-11e8-8bc0-0050569f430b/runs/d020f9c3-252a-41cc-8498-20c8a7e3fba2:/mnt/mesos/sandbox"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"env": "host,hostname,MESOS_TASK_ID,LOGSTASH_TAGS,MESOS_CONTAINER_NAME,MARATHON_APP_ID",
"max-file": "2",
"max-size": "1k"
}
},
"NetworkMode": "macvlan",
"PortBindings": {
"443/tcp": [
{
"HostIp": "",
"HostPort": "56198"
}
],
"80/tcp": [
{
"HostIp": "",
"HostPort": "56197"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": [
"10.62.68.143"
],
"DnsOptions": [
"timeout:2",
"attempts:10"
],
"DnsSearch": [
"lab.nordigy.ru"
],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "shareable",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 134217728,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 430,
"Memory": 268435456,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 536870912,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": [
{
"Name": "nofile",
"Hard": 655350,
"Soft": 655350
},
{
"Name": "memlock",
"Hard": -1,
"Soft": -1
},
{
"Name": "core",
"Hard": -1,
"Soft": -1
},
{
"Name": "stack",
"Hard": -1,
"Soft": -1
},
{
"Name": "nproc",
"Hard": 65535,
"Soft": 65535
}
],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Data": {
"DeviceId": "1908",
"DeviceName": "docker-253:0-265445-7cc268bc75b0ef24c080cad339a8dbc7aea5ab1b927539e633367c4806046e24",
"DeviceSize": "10737418240"
},
"Name": "devicemapper"
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/lib/mesos/slaves/c9fc1b80-454a-4ada-be8b-4d6e770791d6-S1/frameworks/762936cb-22f7-437f-ae8b-eb355e206474-0000/executors/ads80-p10-tst08.eb010f99-64ba-11e8-8bc0-0050569f430b/runs/d020f9c3-252a-41cc-8498-20c8a7e3fba2",
"Destination": "/mnt/mesos/sandbox",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "ad00015b1e67",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": true,
"AttachStderr": true,
"ExposedPorts": {
"443/tcp": {},
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"HOST=ams02-e01-ccs02.lab.nordigy.ru",
"MARATHON_APP_ID=/ads80-p10-tst08",
"PORT=56197",
"PORTS=56197,56198",
"Env_var2=5",
"IsContainer=True",
"MARATHON_APP_RESOURCE_DISK=0.0",
"MESOS_CONTAINER_NAME=mesos-d020f9c3-252a-41cc-8498-20c8a7e3fba2",
"MESOS_SANDBOX=/mnt/mesos/sandbox",
"MESOS_TASK_ID=ads80-p10-tst08.eb010f99-64ba-11e8-8bc0-0050569f430b",
"PORT0=56197",
"Env_var=Env_value",
"MARATHON_APP_RESOURCE_MEM=256.0",
"MARATHON_APP_VERSION=2018-05-15T08:41:05.821Z",
"PORT1=56198",
"PORT_443=56198",
"LOGSTASH_TAGS=ads80-p10-tst08",
"MARATHON_APP_LABELS=",
"MARATHON_APP_RESOURCE_CPUS=0.42",
"MARATHON_APP_RESOURCE_GPUS=0",
"PORT_80=56197",
"SERVICE_NAME=ads80-p10-tst08",
"MARATHON_APP_DOCKER_IMAGE=docker-registry.lab.nordigy.ru:443/rc_ads/ras:latest",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/opt/python/latest/bin/gunicorn",
"ras.wsgi",
"-b",
"0.0.0.0:80",
"-w",
"2"
],
"ArgsEscaped": true,
"Image": "docker-registry.lab.nordigy.ru:443/rc_ads/ras:latest",
"Volumes": null,
"WorkingDir": "/opt/ras",
"Entrypoint": [
"/usr/local/bin/entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"MESOS_TASK_ID": "ads80-p10-tst08.eb010f99-64ba-11e8-8bc0-0050569f430b"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "06d08979106d9a1d500b8592f8ab4092e3b5947c07f822a4b9d43e0355a7a903",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/06d08979106d",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"macvlan": {
"IPAMConfig": null,
"Links": null,
"Aliases": [
"ad00015b1e67"
],
"NetworkID": "06a1e9329efcae7d6cd767960b09abe19c8a1cda10cd4baf8f98611a5eb7caf8",
"EndpointID": "c7932163ff578c49a3d9efeb8e2cd1e408b9112ce8795152fced1a587c2e19ef",
"Gateway": "10.62.64.1",
"IPAddress": "10.62.92.5",
"IPPrefixLen": 19,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"DriverOpts": null
}
}
}
}
]
# docker exec -ti ad00015b1e67 cat /etc/resolv.conf
search lab.nordigy.ru
nameserver 127.0.0.11
options timeout:2 attempts:10 ndots:0
It still using embedded DNS.
Steps to reproduce the behavior
Run container with macvlan/ipvlan network
# docker network inspect macvlan
[
{
"Name": "macvlan",
"Id": "06a1e9329efcae7d6cd767960b09abe19c8a1cda10cd4baf8f98611a5eb7caf8",
"Created": "2018-05-14T11:52:09.484119132Z",
"Scope": "local",
"Driver": "ipvlan",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": {},
"Config": [
{
"Subnet": "10.62.64.0/19",
"IPRange": "10.62.92.4/30",
"Gateway": "10.62.64.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"480d10fb3dab6665e7d654a00ed57fc6e5b7258b5496a20c9174553df57726c3": {
"Name": "mesos-39de5374-9768-4d29-a4d1-ac9c7b22ff65",
"EndpointID": "ab2bcbc8acaa5826c9b3488f292d9a3930fdd97533b00183cc79011036c34e11",
"MacAddress": "",
"IPv4Address": "10.62.92.4/19",
"IPv6Address": ""
},
"ad00015b1e67382b27054bd243c96dd1fd4f14994eb12be23c06d3ee715b179c": {
"Name": "mesos-d020f9c3-252a-41cc-8498-20c8a7e3fba2",
"EndpointID": "c7932163ff578c49a3d9efeb8e2cd1e408b9112ce8795152fced1a587c2e19ef",
"MacAddress": "",
"IPv4Address": "10.62.92.5/19",
"IPv6Address": ""
}
},
"Options": {
"parent": "eth0"
},
"Labels": {}
}
]
Output of docker version:
# docker version
Client:
Version: 18.03.1-ce
API version: 1.37
Go version: go1.9.5
Git commit: 9ee9f40
Built: Thu Apr 26 07:20:16 2018
OS/Arch: linux/amd64
Experimental: false
Orchestrator: swarm
Server:
Engine:
Version: 18.03.1-ce
API version: 1.37 (minimum version 1.12)
Go version: go1.9.5
Git commit: 9ee9f40
Built: Thu Apr 26 07:23:58 2018
OS/Arch: linux/amd64
Experimental: true
Output of docker info:
# docker info
Containers: 13
Running: 7
Paused: 0
Stopped: 6
Images: 4
Server Version: 18.03.1-ce
Storage Driver: devicemapper
Pool Name: vg_docker-lv_docker
Pool Blocksize: 65.54kB
Base Device Size: 10.74GB
Backing Filesystem: xfs
Udev Sync Supported: true
Data Space Used: 3.158GB
Data Space Total: 123.3GB
Data Space Available: 120.2GB
Metadata Space Used: 5.657MB
Metadata Space Total: 2.751GB
Metadata Space Available: 2.746GB
Thin Pool Minimum Free Space: 12.33GB
Deferred Removal Enabled: false
Deferred Deletion Enabled: false
Deferred Deleted Device Count: 0
Library Version: 1.02.107-RHEL7 (2015-10-14)
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
Volume: local
Network: bridge host ipvlan macvlan null overlay
Log: awslogs fluentd gcplogs gelf journald json-file logentries splunk syslog
Swarm: inactive
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: 209a7fc3e4a32ef71a8c7b50c68fc8398415badf (expected: 773c489c9c1b21a6d78b5c538cd395416ec50f88)
runc version: 4fc53a81fb7c994640722ac585fa9ca548971871
init version: 949e6fa
Security Options:
seccomp
Profile: default
Kernel Version: 4.9.98-1.el7.centos.x86_64
Operating System: CentOS Linux 7 (Core)
OSType: linux
Architecture: x86_64
CPUs: 4
Total Memory: 15.66GiB
Name: ams02-e01-ccs02
ID: 5KPN:EASA:7BRP:Q6SU:EOFF:YJH7:XOMD:YJPA:WV2A:4XXI:EDYE:34RG
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): true
File Descriptors: 156
Goroutines: 197
System Time: 2018-05-31T10:21:03.802131982Z
EventsListeners: 2
Registry: https://index.docker.io/v1/
Labels:
Experimental: true
Insecure Registries:
127.0.0.0/8
Live Restore Enabled: false
Additional environment details (AWS, VirtualBox, physical, etc.)
I have same problem.
Steps:
- create a container with
--dns
# docker run -d -t --name t1 --dns="223.5.5.5" busybox
538b414aa29738752e8f8b6e5e9c2ce40347d9195a4f1389ea5d7d1dbbf5d9c0
- get the resolv.conf
# docker exec 53 cat /etc/resolv.conf
nameserver 223.5.5.5
- create a network
# docker network create testNet1 --subnet 111.0.0.0/24
1fb10ce7f43b1a330f8d0218851364b77c9a1db52b1ddf9fef0335ae11f5c0c1
- connect the network to the container
# docker network connect testNet1 t1
- show the resolv.conf again
# docker exec 53 cat /etc/resolv.conf
nameserver 127.0.0.11
options ndots:0
some informations
# docker -v
Docker version 18.03.1-ce, build 9ee9f40
# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 16.04.4 LTS
Release: 16.04
Codename: xenial
# uname -a
Linux slt-docker 4.4.0-128-generic #154-Ubuntu SMP Fri May 25 14:15:18 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
This is the expected behaviour; when using a custom network, the embedded DNS is used to resolve other services or containers on the network. The embedded DNS forwards other DNS requests to the configured external DNS servers.
If you enable debug mode on the daemon, you can see messages in the daemon logs that show it's resolving using the external DNS.
DEBU[2018-06-14T20:34:39.649750900Z] Name To resolve: google.com.
DEBU[2018-06-14T20:34:39.649954200Z] [resolver] query google.com. (A) from 172.19.0.2:48494, forwarding to udp:192.168.65.1
DEBU[2018-06-14T20:34:39.650351300Z] Name To resolve: google.com.
DEBU[2018-06-14T20:34:39.650587800Z] [resolver] query google.com. (AAAA) from 172.19.0.2:39164, forwarding to udp:192.168.65.1
DEBU[2018-06-14T20:34:39.656789400Z] [resolver] received A record "216.58.194.174" for "google.com." from udp:192.168.65.1
DEBU[2018-06-14T20:34:39.705113800Z] [resolver] received AAAA record "2607:f8b0:4005:802::200e" for "google.com." from udp:192.168.65.1
Thanks for your info @thaJeztah
The embedded DNS forwards other DNS requests to the configured external DNS servers.
Is the external DNS servers are /etc/resolv.conf or 'dns' for /etc/docker/daemon.json?
By default it should use the DNS that's configured on the host, which can be overridden in the daemon.json, but you can see the messages in the daemon logs of debug is enabled
But, sometimes embedded DNS haven't forward request to the configured DNS..
What do the logs show in that case (with debug enabled)? "sometimes" is a bit vague
@thaJeztah What should we do when using different DNS in different containers?
@wkite not sure I understand your question; could you explain? You can set a different DNS on each container if you want to (i.e., docker run --dns=1.1.1.1 ....), the embedded DNS will forward requests to that DNS if it's not an internal entry.
@thaJeztah I need to use internal DNS in container with custom network, but the --dns option doesn't work. What should I do? thx.
but the --dns option doesn't work
@wkite what doesn't work? Can you show what you mean with that?
--dns=[IP_ADDRESS...] The IP addresses passed via the --dns option is used by the embedded DNS server to forward the DNS query if embedded DNS server is unable to resolve a name resolution request from the containers. These --dns IP addresses are managed by the embedded DNS server and will not be updated in the container's /etc/resolv.conf file.
I ran into the same issue, Wireshark helps understand how queries actually flow. But it would be really nice that we can make it do what we actually configured for the DNS servers. In case the user does not want Docker internal DNS or needs to override it completely
