registrator
registrator copied to clipboard
gliderlabs
Use ip from env
Extending the support to pull request https://github.com/gliderlabs/registrator/pull/544 The Registrator will register services to backend using run-time IPs provided by Kubernetes. Since EKS will use a non-bridge approach i.e. preallocating IPs on Worker-Nodes, we need to have a way to pass dynamic IPs to Registrator. In Kubernetes, we can use downward status APIs to get POD ips at run time, and we can assign an environment variable to them.
Example:
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
With this in mind, we can run registrator like a daemon set, while consul-client is also running as daemonset in Kubernetes.
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
run: registrator
name: registrator
spec:
selector:
matchLabels:
run: registrator
template:
metadata:
creationTimestamp: null
labels:
run: registrator
app: registrator
service: consul-registrator
department: cloudops
spec:
hostNetwork: true
containers:
- image: artifactory.arlocloud.com/docker-local/registrator:v4
name: registrator
command: ["/bin/sh"]
args: ["-c", "registrator -useIpFromEnv=POD_IP -explicit=true -resync=60 -cleanup -deregister=always consul://${NODE_IP}:8500"]
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- mountPath: /tmp/docker.sock
name: docker-sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
Now deployment spec can be changed to below to provide auto-registration:
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx0-deployment
labels:
app: nginx0-deployment
spec:
replicas: 2
selector:
matchLabels:
app: nginx0
template:
metadata:
labels:
app: nginx0
spec:
containers:
- name: nginx
image: k8s.gcr.io/nginx:1.7.9
ports:
- containerPort: 80
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SERVICE_80_NAME
value: nginx
- name: SERVICE_PORT
value: "80"
- name: SERVICE_TAGS
value: "nginx,eu-west-1"
resources:
requests:
memory: "250Mi"
cpu: "500m"
limits:
memory: "250Mi"
cpu: "500m"
One valid container spec using docker inspect for validation:
[
{
"Id": "4ca22d6d7b1ed00c987285273689fd67a304c59a5ed8678ce8e3ccfd76b12244",
"Created": "2020-01-10T14:20:04.909668018Z",
"Path": "nginx",
"Args": [
"-g",
"daemon off;"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 5050,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-01-10T14:20:05.108602423Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:6b60a9253836f5812ef4567776db8bf88f5996f0883bfde6ed6c0ccd02979f08",
"ResolvConfPath": "/var/lib/docker/containers/85806b20ead4008d9f0a7cd1711ebe22b748820fad372df5e8dee6a1abf9c0f1/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/85806b20ead4008d9f0a7cd1711ebe22b748820fad372df5e8dee6a1abf9c0f1/hostname",
"HostsPath": "/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/etc-hosts",
"LogPath": "/var/lib/docker/containers/4ca22d6d7b1ed00c987285273689fd67a304c59a5ed8678ce8e3ccfd76b12244/4ca22d6d7b1ed00c987285273689fd67a304c59a5ed8678ce8e3ccfd76b12244-json.log",
"Name": "/k8s_nginx_nginx0-deployment-677bc9b44b-55cvn_consul-client_4b5949dd-33b4-11ea-9080-068ce2d35fbe_0",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/volumes/kubernetes.io~secret/default-token-kcdwp:/var/run/secrets/kubernetes.io/serviceaccount:ro",
"/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/etc-hosts:/etc/hosts",
"/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/containers/nginx/5d0e8784:/dev/termination-log"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "10",
"max-size": "10m"
}
},
"NetworkMode": "container:85806b20ead4008d9f0a7cd1711ebe22b748820fad372df5e8dee6a1abf9c0f1",
"PortBindings": null,
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "container:85806b20ead4008d9f0a7cd1711ebe22b748820fad372df5e8dee6a1abf9c0f1",
"Cgroup": "",
"Links": null,
"OomScoreAdj": -998,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 512,
"Memory": 262144000,
"NanoCpus": 0,
"CgroupParent": "/kubepods/pod4b5949dd-33b4-11ea-9080-068ce2d35fbe",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 100000,
"CpuQuota": 50000,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 262144000,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/asound",
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/8857ff904d0822daa2ec50ebc5aa23fbabfa1fa0f9d644baaec3087debac999d-init/diff:/var/lib/docker/overlay2/9b7eaecedf5fc0f1424e2f33da9c960572df5d310d337666a1566bf4b8ae3b1e/diff:/var/lib/docker/overlay2/455aa47467b133baa5151b7951f023605dad8b6c46a19d024562dc060c9d70e1/diff:/var/lib/docker/overlay2/06d0b5b0585cc96d5f97a87d17af845037cd6541e94ed42b1f0736ac87829df1/diff:/var/lib/docker/overlay2/b1d44cdd08ea0ec08e31c2e8b6a908893ca99288d18638da49193e737b4d30d2/diff:/var/lib/docker/overlay2/b77c69ca1c609f4981a6692863b6d0b6b700c681948814bc27b40f7d08a9278a/diff:/var/lib/docker/overlay2/a872afabaf1c13915442a1d7814500137cc09132a4762fdfd04d3eebb0f16be7/diff:/var/lib/docker/overlay2/63bb79425b96a0ae54ecf4e8b460409f40bd25a53366ae7cc230147c1c5af73c/diff:/var/lib/docker/overlay2/97a4af225e705cb9f90d814f077b48e7280529141eef6ea317fe2cc6c6603abd/diff:/var/lib/docker/overlay2/236fd65f097438cf7a9cc9d229db1275e71748f41c422b6e3fc9c711bea2b3c9/diff:/var/lib/docker/overlay2/53e48c96aa8be9ffd3f7c69b1407cda50e67fdf290d16d04485dffd572c90991/diff:/var/lib/docker/overlay2/6b146c31097f1e009f64f7d8a2288146121fbf6ef7e30d026aa6e859cdcbc329/diff:/var/lib/docker/overlay2/582abe2ad99830bb5e1ce29095ab64c26158e939cfc8972b85ccf23a8ff4aa3a/diff:/var/lib/docker/overlay2/349c452d9f883bffeb5103fff88f3ffda8e2570a253405663e5da2b4fd5c2101/diff",
"MergedDir": "/var/lib/docker/overlay2/8857ff904d0822daa2ec50ebc5aa23fbabfa1fa0f9d644baaec3087debac999d/merged",
"UpperDir": "/var/lib/docker/overlay2/8857ff904d0822daa2ec50ebc5aa23fbabfa1fa0f9d644baaec3087debac999d/diff",
"WorkDir": "/var/lib/docker/overlay2/8857ff904d0822daa2ec50ebc5aa23fbabfa1fa0f9d644baaec3087debac999d/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/containers/nginx/5d0e8784",
"Destination": "/dev/termination-log",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "ad71247169c4802e2105a694c650db48e6a40e92d4ee0bdfc9a57e78b026bf22",
"Source": "/var/lib/docker/volumes/ad71247169c4802e2105a694c650db48e6a40e92d4ee0bdfc9a57e78b026bf22/_data",
"Destination": "/var/cache/nginx",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/volumes/kubernetes.io~secret/default-token-kcdwp",
"Destination": "/var/run/secrets/kubernetes.io/serviceaccount",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/4b5949dd-33b4-11ea-9080-068ce2d35fbe/etc-hosts",
"Destination": "/etc/hosts",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "nginx0-deployment-677bc9b44b-55cvn",
"Domainname": "",
"User": "0",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"443/tcp": {},
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"POD_IP=172.24.65.84",
"NAMESPACE=consul-client",
"SERVICE_80_NAME=nginx",
"SERVICE_PORT=80",
"SERVICE_TAGS=nginx,eu-west-1",
"NODE_NAME=ip-172-24-65-60.eu-west-1.compute.internal",
"NODE_IP=172.24.65.60",
"KUBERNETES_SERVICE_PORT=443",
"KUBERNETES_SERVICE_PORT_HTTPS=443",
"KUBERNETES_PORT=tcp://10.100.0.1:443",
"KUBERNETES_PORT_443_TCP=tcp://10.100.0.1:443",
"KUBERNETES_PORT_443_TCP_PROTO=tcp",
"KUBERNETES_PORT_443_TCP_PORT=443",
"KUBERNETES_PORT_443_TCP_ADDR=10.100.0.1",
"KUBERNETES_SERVICE_HOST=10.100.0.1",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"NGINX_VERSION=1.7.9-1~wheezy"
],
"Cmd": [
"nginx",
"-g",
"daemon off;"
],
"Healthcheck": {
"Test": [
"NONE"
]
},
"Image": "sha256:6b60a9253836f5812ef4567776db8bf88f5996f0883bfde6ed6c0ccd02979f08",
"Volumes": {
"/var/cache/nginx": {}
},
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
"annotation.io.kubernetes.container.hash": "49d274c8",
"annotation.io.kubernetes.container.ports": "[{\"containerPort\":80,\"protocol\":\"TCP\"}]",
"annotation.io.kubernetes.container.restartCount": "0",
"annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
"annotation.io.kubernetes.container.terminationMessagePolicy": "File",
"annotation.io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.container.logpath": "/var/log/pods/consul-client_nginx0-deployment-677bc9b44b-55cvn_4b5949dd-33b4-11ea-9080-068ce2d35fbe/nginx/0.log",
"io.kubernetes.container.name": "nginx",
"io.kubernetes.docker.type": "container",
"io.kubernetes.pod.name": "nginx0-deployment-677bc9b44b-55cvn",
"io.kubernetes.pod.namespace": "consul-client",
"io.kubernetes.pod.uid": "4b5949dd-33b4-11ea-9080-068ce2d35fbe",
"io.kubernetes.sandbox.id": "85806b20ead4008d9f0a7cd1711ebe22b748820fad372df5e8dee6a1abf9c0f1"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]