tox-docker
tox-docker copied to clipboard
Published
20 hours ago •
tox-dev
tox-dev
Container never becomes healthy
I'm using this code on my tox.ini:
[docker:postgres]
image=postgres:13
environment=
POSTGRES_PASSWORD=postgres
ports=5432:5432/tcp
healthcheck_cmd = pg_isready --username postgres
healthcheck_retries = 30
healthcheck_timeout = 1
healthcheck_interval = 1
healthcheck_start_period = 1
And the container is never reported as healthy:
django32-postgres docker: run 'postgres:13' (from 'postgres')
django32-postgres docker: health check 'docker.io/library/postgres:13' (from 'postgres')
django32-postgres docker: leave 'f45481a0ec' (from 'postgres') running
___________________________________________________________________ summary ___________________________________________________________________
ERROR: django32-postgres: 'docker.io/library/postgres:13' (from 'postgres') failed health check
Traceback (most recent call last):
File "/usr/bin/tox", line 33, in <module>
sys.exit(load_entry_point('tox==3.24.1', 'console_scripts', 'tox')())
File "/usr/lib/python3.9/site-packages/tox/session/__init__.py", line 44, in cmdline
main(args)
File "/usr/lib/python3.9/site-packages/tox/session/__init__.py", line 69, in main
exit_code = session.runcommand()
File "/usr/lib/python3.9/site-packages/tox/session/__init__.py", line 197, in runcommand
return self.subcommand_test()
File "/usr/lib/python3.9/site-packages/tox/session/__init__.py", line 225, in subcommand_test
run_sequential(self.config, self.venv_dict)
File "/usr/lib/python3.9/site-packages/tox/session/commands/run/sequential.py", line 22, in run_sequential
runtestenv(venv, config)
File "/usr/lib/python3.9/site-packages/tox/session/commands/run/sequential.py", line 73, in runtestenv
config.pluginmanager.hook.tox_runtest_pre(venv=venv)
File "/usr/lib/python3.9/site-packages/pluggy/hooks.py", line 286, in __call__
return self._hookexec(self, self.get_hookimpls(), kwargs)
File "/usr/lib/python3.9/site-packages/pluggy/manager.py", line 93, in _hookexec
return self._inner_hookexec(hook, methods, kwargs)
File "/usr/lib/python3.9/site-packages/pluggy/manager.py", line 84, in <lambda>
self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(
File "/usr/lib/python3.9/site-packages/pluggy/callers.py", line 208, in _multicall
return outcome.get_result()
File "/usr/lib/python3.9/site-packages/pluggy/callers.py", line 80, in get_result
raise ex[1].with_traceback(ex[2])
File "/usr/lib/python3.9/site-packages/pluggy/callers.py", line 187, in _multicall
res = hook_impl.function(*args)
File "/usr/lib/python3.9/site-packages/tox_docker/__init__.py", line 338, in tox_runtest_pre
raise HealthCheckFailed(msg)
tox_docker.HealthCheckFailed: 'docker.io/library/postgres:13' (from 'postgres') failed health check
Here's the output of docker inspect...:
[
{
"Id": "c1caad714f8e91bd9e4b352dcaac9ed91cde2f4a68fc674b65928cebb5f5239c",
"Created": "2021-08-08T09:54:23.783351188Z",
"Path": "docker-entrypoint.sh",
"Args": [
"postgres"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 1331140,
"ExitCode": 0,
"Error": "",
"StartedAt": "2021-08-08T09:54:24.008813678Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Health": {
"Status": "starting",
"FailingStreak": 15,
"Log": [
{
"Start": "2021-08-08T11:54:47.42803867+02:00",
"End": "2021-08-08T11:54:47.544048527+02:00",
"ExitCode": 1,
"Output": ""
},
{
"Start": "2021-08-08T11:54:49.432903825+02:00",
"End": "2021-08-08T11:54:49.505735305+02:00",
"ExitCode": 1,
"Output": ""
},
{
"Start": "2021-08-08T11:54:51.467404167+02:00",
"End": "2021-08-08T11:54:51.542379598+02:00",
"ExitCode": 1,
"Output": ""
},
{
"Start": "2021-08-08T11:54:53.471339031+02:00",
"End": "2021-08-08T11:54:53.583864832+02:00",
"ExitCode": 1,
"Output": ""
},
{
"Start": "2021-08-08T11:54:55.410558807+02:00",
"End": "2021-08-08T11:54:55.486763581+02:00",
"ExitCode": 1,
"Output": ""
}
]
}
},
"Image": "docker.io/library/postgres:13",
"ResolvConfPath": "/run/user/1000/containers/overlay-containers/c1caad714f8e91bd9e4b352dcaac9ed91cde2f4a68fc674b65928cebb5f5239c/userdata/resolv.conf",
"HostnamePath": "/run/user/1000/containers/overlay-containers/c1caad714f8e91bd9e4b352dcaac9ed91cde2f4a68fc674b65928cebb5f5239c/userdata/hostname",
"HostsPath": "/run/user/1000/containers/overlay-containers/c1caad714f8e91bd9e4b352dcaac9ed91cde2f4a68fc674b65928cebb5f5239c/userdata/hosts",
"LogPath": "/home/hugo/.local/share/containers/storage/overlay-containers/c1caad714f8e91bd9e4b352dcaac9ed91cde2f4a68fc674b65928cebb5f5239c/userdata/ctr.log",
"Name": "/postgres",
"RestartCount": 0,
"Driver": "overlay",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": [],
"HostConfig": {
"Binds": [
"879868bc4ed36c9ddb38b16222bebabe2ce48ac60dc55ca97d739b17ee3354ea:/var/lib/postgresql/data:rprivate,rw,nodev,exec,nosuid,rbind"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": null
},
"NetworkMode": "slirp4netns",
"PortBindings": {
"5432/tcp": [
{
"HostIp": "",
"HostPort": "5432"
}
]
},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": [],
"CapDrop": [
"AUDIT_WRITE",
"MKNOD",
"NET_RAW"
],
"CgroupnsMode": "",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": [],
"GroupAdd": [],
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "private",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [],
"UTSMode": "private",
"UsernsMode": "",
"ShmSize": 65536000,
"Runtime": "oci",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "user.slice",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": 0,
"OomKillDisable": false,
"PidsLimit": 2048,
"Ulimits": [],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"LowerDir": "/home/hugo/.local/share/containers/storage/overlay/35591b3f3bea9079c011da63b1a129b81957a3631ff2ce4ac6e892af82255bdf/diff:/home/hugo/.local/share/containers/storage/overlay/ceb052db2a6c7d55a47c596c811138e02ee2d9a814481098350de9c1f16f3264/diff:/home/hugo/.local/share/containers/storage/overlay/3ac859ffc5aa6ce7e61b219946484f1369aba3009104eaa31764b28711383702/diff:/home/hugo/.local/share/containers/storage/overlay/3b4d7cd67336eed72526a139f0a353eef25cb6d090388bac0d9a0cdff34b2998/diff:/home/hugo/.local/share/containers/storage/overlay/5a10d6774b8456893ebc3f3a2ef05ccfdfae0a260b7777f921c2de586dbdb6e8/diff:/home/hugo/.local/share/containers/storage/overlay/feb8c65ded5e1ae41255dac9afa61bcb6dc4587f9111e64f4198cf5406e38eb9/diff:/home/hugo/.local/share/containers/storage/overlay/06b049fc1e45f8ab6160f1639d4ffd0e8477f8a7841cfd54ae338a113320d7fc/diff:/home/hugo/.local/share/containers/storage/overlay/52ce6081a1d204b1a757e3ecfcd69011261d9d8a43ea33113809b5db0bbb9827/diff:/home/hugo/.local/share/containers/storage/overlay/194f232660e3a1749dd620b226d1815ad95697ab35c5726fc5d1cbacacd8818d/diff:/home/hugo/.local/share/containers/storage/overlay/63d68fc67685c412bff4fd6e89f0c7db1e7bfd6512ef8f2243955d8d8fe9cf0a/diff:/home/hugo/.local/share/containers/storage/overlay/cd3d341c687e1bfa84a81c4e38ca59f9925790c71804701e5c98a564363c387f/diff:/home/hugo/.local/share/containers/storage/overlay/f2640ac368df8cfd4af9ecc847a3782830026f13da54a5b6539633745810e167/diff:/home/hugo/.local/share/containers/storage/overlay/814bff7343242acfd20a2c841e041dd57c50f0cf844d4abd2329f78b992197f4/diff",
"MergedDir": "/home/hugo/.local/share/containers/storage/overlay/229dd015c29dede04b0cb157a15b013dcd28ecf3e4bf170543895018c05581be/merged",
"UpperDir": "/home/hugo/.local/share/containers/storage/overlay/229dd015c29dede04b0cb157a15b013dcd28ecf3e4bf170543895018c05581be/diff",
"WorkDir": "/home/hugo/.local/share/containers/storage/overlay/229dd015c29dede04b0cb157a15b013dcd28ecf3e4bf170543895018c05581be/work"
},
"Name": "overlay"
},
"SizeRootFs": 0,
"Mounts": [
{
"Type": "volume",
"Name": "879868bc4ed36c9ddb38b16222bebabe2ce48ac60dc55ca97d739b17ee3354ea",
"Source": "/home/hugo/.local/share/containers/storage/volumes/879868bc4ed36c9ddb38b16222bebabe2ce48ac60dc55ca97d739b17ee3354ea/_data",
"Destination": "/var/lib/postgresql/data",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "c1caad714f8e",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"5432/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/postgresql/13/bin",
"TERM=xterm",
"container=podman",
"PG_MAJOR=13",
"PG_VERSION=13.3-1.pgdg100+1",
"GOSU_VERSION=1.12",
"LANG=en_US.utf8",
"PGDATA=/var/lib/postgresql/data",
"POSTGRES_PASSWORD=postgres",
"HOME=/root",
"HOSTNAME=c1caad714f8e"
],
"Cmd": [
"postgres"
],
"Image": "docker.io/library/postgres:13",
"Volumes": null,
"WorkingDir": "/",
"Entrypoint": [
"docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"tox_docker_container_name": "postgres"
},
"StopSignal": "2",
"StopTimeout": 0
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"5432/tcp": [
{
"HostIp": "",
"HostPort": "5432"
}
]
},
"SandboxKey": "/run/user/1000/netns/cni-262db29d-76da-0e2a-36d0-7e1968570ad6",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]
Using docker exec -it c1 pg_isready --username postgres exits zero. (and prints /var/run/postgresql:5432 - accepting connections).
Any ideas what I'm doing wrong?
Maybe you need to specify the full path to pg_isready? I'm not sure what PATH exists when health checks are run (possibly it's empty).
If that fails, you could try docker run -e POSTGRES_PASSWORD=postgres -t -i --health-start-period 1s --health-interval 1s --health-timeout 1s --health-retries 30 --health-cmd "pg_isready --username postgres" postgres:13 to debug...
That should be equivalent to what tox-docker is running.
