dask-kubernetes
dask-kubernetes copied to clipboard
Published
20 hours ago •
dask
dask
Allow graceful shutdown
Kubecluster fails on shutdown with the error below.
Exception ignored in atexit callback: <function reap_clusters at 0x7ffb46fd65c0>
Traceback (most recent call last):
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/dask_kubernetes/operator/kubecluster/kubecluster.py", line 1025, in reap_clusters
asyncio.run(_reap_clusters())
File "/usr/lib/python3.11/asyncio/runners.py", line 190, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/asyncio/base_events.py", line 653, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/dask_kubernetes/operator/kubecluster/kubecluster.py", line 1023, in _reap_clusters
cluster.close(timeout=10)
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/dask_kubernetes/operator/kubecluster/kubecluster.py", line 692, in close
return self.sync(self._close, timeout=timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/distributed/utils.py", line 357, in sync
return sync(
^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/distributed/utils.py", line 424, in sync
raise exc.with_traceback(tb)
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/distributed/utils.py", line 397, in f
result = yield future
^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/tornado/gen.py", line 767, in run
value = future.result()
^^^^^^^^^^^^^^^
File "/mnt/e/Dev/raven/raven/core/cluster/kubernetes.py", line 108, in _close
return await super()._close(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/dask_kubernetes/operator/kubecluster/kubecluster.py", line 698, in _close
cluster = await DaskCluster.get(self.name, namespace=self.namespace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_objects.py", line 177, in get
resources = await api._get(
^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_api.py", line 323, in _get
async with self._get_kind(
File "/usr/lib/python3.11/contextlib.py", line 204, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_api.py", line 252, in _get_kind
async with self.call_api(
File "/usr/lib/python3.11/contextlib.py", line 204, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_api.py", line 133, in call_api
response = await self._session.request(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpx/_client.py", line 1530, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpx/_client.py", line 1617, in send
response = await self._send_handling_auth(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpx/_client.py", line 1645, in _send_handling_auth
response = await self._send_handling_redirects(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpx/_client.py", line 1682, in _send_handling_redirects
response = await self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpx/_client.py", line 1719, in _send_single_request
response = await transport.handle_async_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 366, in handle_async_request
resp = await self._pool.handle_async_request(req)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 262, in handle_async_request
raise exc
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 245, in handle_async_request
response = await connection.handle_async_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 99, in handle_async_request
raise exc
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 76, in handle_async_request
stream = await self._connect(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 124, in _connect
stream = await self._network_backend.connect_tcp(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_backends/auto.py", line 31, in connect_tcp
return await self._backend.connect_tcp(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/httpcore/_backends/anyio.py", line 114, in connect_tcp
stream: anyio.abc.ByteStream = await anyio.connect_tcp(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/anyio/_core/_sockets.py", line 193, in connect_tcp
gai_res = await getaddrinfo(
^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/anyio/_core/_sockets.py", line 568, in getaddrinfo
gai_res = await get_async_backend().getaddrinfo(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 2304, in getaddrinfo
return await get_running_loop().getaddrinfo(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/asyncio/base_events.py", line 867, in getaddrinfo
return await self.run_in_executor(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/asyncio/base_events.py", line 829, in run_in_executor
executor.submit(func, *args), loop=self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/concurrent/futures/thread.py", line 167, in submit
raise RuntimeError('cannot schedule new futures after shutdown')
RuntimeError: cannot schedule new futures after shutdown
Task exception was never retrieved
future: <Task finished name='Task-30' coro=<PortForward._sync_sockets() done, defined at /root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_portforward.py:167> exception=ExceptionGroup('unhandled errors in a TaskGroup', [ConnectionClosedError('TCP socket closed')])>
+ Exception Group Traceback (most recent call last):
| File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_portforward.py", line 171, in _sync_sockets
| async with anyio.create_task_group() as tg:
| File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 664, in __aexit__
| raise BaseExceptionGroup(
| ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_portforward.py", line 183, in _tcp_to_ws
| raise ConnectionClosedError("TCP socket closed")
| kr8s._exceptions.ConnectionClosedError: TCP socket closed
+------------------------------------
Task exception was never retrieved
future: <Task finished name='Task-41' coro=<PortForward._sync_sockets() done, defined at /root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_portforward.py:167> exception=ExceptionGroup('unhandled errors in a TaskGroup', [ConnectionClosedError('TCP socket closed')])>
+ Exception Group Traceback (most recent call last):
| File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_portforward.py", line 171, in _sync_sockets
| async with anyio.create_task_group() as tg:
| File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 664, in __aexit__
| raise BaseExceptionGroup(
| ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/root/.cache/pypoetry/virtualenvs/raven-Io4q2IrD-py3.11/lib/python3.11/site-packages/kr8s/_portforward.py", line 183, in _tcp_to_ws
| raise ConnectionClosedError("TCP socket closed")
| kr8s._exceptions.ConnectionClosedError: TCP socket closed
+------------------------------------
