cog
cog copied to clipboard
replicate
Unhandled InvalidStateException crashes the http server:
Our logs suggest that a potentially unhandled InvalidStateException can crash the container:
Invalid operation: state is WorkerState.PROCESSING (must be WorkerState.READY)
To my understanding, it indicates that the application is trying to run a prediction while the worker is in a PROCESSING state. The raised InvalidStateException does not seem to be handled gracefully by the http server.
[
{
"logger": "cog.server.runner",
"timestamp": "2024-02-14T11:03:03.228760Z",
"exception": "Traceback (most recent call last):\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/runner.py\", line 134, in handle_error\n
raise error\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/multiprocessing/pool.py\", line 125, in worker\n
result = (True, func(*args, **kwds))\n
^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/runner.py\", line 359, in predict\n
return _predict(\n
^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/runner.py\", line 395, in _predict\n
for event in worker.predict(input_dict, poll=0.1):\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/worker.py\", line 63, in predict\n
self._assert_state(WorkerState.READY)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/worker.py\", line 97, in _assert_state\n
raise InvalidStateException(\n
cog.server.exceptions.InvalidStateException: Invalid operation: state is WorkerState.PROCESSING (must be WorkerState.READY)",
"severity": "ERROR",
"message": "caught exception while running prediction"
},
{
"prediction_id": null,
"logger": "uvicorn.error",
"timestamp": "2024-02-14T11:03:03.229663Z",
"exception": "Traceback (most recent call last):\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py\", line 419, in run_asgi\n
result = await app( # type: ignore[func-returns-value]\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py\", line 84, in __call__\n
return await self.app(scope, receive, send)\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/fastapi/applications.py\", line 284, in __call__\n
await super().__call__(scope, receive, send)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/applications.py\", line 122, in __call__\n
await self.middleware_stack(scope, receive, send)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/middleware/errors.py\", line 184, in __call__\n
raise exc\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/middleware/errors.py\", line 162, in __call__\n
await self.app(scope, receive, _send)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/middleware/exceptions.py\", line 79, in __call__\n
raise exc\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/middleware/exceptions.py\", line 68, in __call__\n
await self.app(scope, receive, sender)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py\", line 20, in __call__\n
raise e\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py\", line 17, in __call__\n
await self.app(scope, receive, send)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/routing.py\", line 718, in __call__\n
await route.handle(scope, receive, send)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/routing.py\", line 276, in handle\n
await self.app(scope, receive, send)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/starlette/routing.py\", line 66, in app\n
response = await func(request)\n
^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/fastapi/routing.py\", line 241, in app\n
raw_response = await run_endpoint_function(\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/fastapi/routing.py\", line 167, in run_endpoint_function\n
return await dependant.call(**values)\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/http.py\", line 271, in predict\n
return _predict(request=request, respond_async=respond_async)\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/http.py\", line 337, in _predict\n
response = PredictionResponse(**async_result.get().dict())\n
^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/multiprocessing/pool.py\", line 774, in get\n
raise self._value\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/runner.py\", line 134, in handle_error\n
raise error\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/multiprocessing/pool.py\", line 125, in worker\n
result = (True, func(*args, **kwds))\n
^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/runner.py\", line 359, in predict\n
return _predict(\n
^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/runner.py\", line 395, in _predict\n
for event in worker.predict(input_dict, poll=0.1):\n
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/worker.py\", line 63, in predict\n
self._assert_state(WorkerState.READY)\n
File \"/root/.pyenv/versions/3.11.8/lib/python3.11/site-packages/cog/server/worker.py\", line 97, in _assert_state\n
raise InvalidStateException(\n
cog.server.exceptions.InvalidStateException: Invalid operation: state is WorkerState.PROCESSING (must be WorkerState.READY)",
"severity": "ERROR",
"message": "Exception in ASGI application\n
"
}
]
GPU inference, 1 uvicorn worker, 1 thread.
