Instance - Stop command return connectionError

Open rapcmia opened this issue 8 months ago • 0 comments

Describe the bug

When running instances and checking pause and stop buttons, the stop button cause connectionError and does not stop the hummingbot container

JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Traceback:
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/streamlit/runtime/scriptrunner/script_runner.py", line 562, in _run_script
    self._session_state.on_script_will_rerun(
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/streamlit/runtime/state/safe_session_state.py", line 65, in on_script_will_rerun
    self._state.on_script_will_rerun(latest_widget_states)
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/streamlit/runtime/state/session_state.py", line 514, in on_script_will_rerun
    self._call_callbacks()
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/streamlit/runtime/state/session_state.py", line 527, in _call_callbacks
    self._new_widget_state.call_callback(wid)
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/streamlit/runtime/state/session_state.py", line 271, in call_callback
    callback(*args, **kwargs)
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/streamlit_elements/core/callback.py", line 91, in dispatch
    callback(**frontend_params)
File "/home/dashboard/frontend/components/bot_performance_card.py", line 167, in <lambda>
    onClick=lambda: stop_bot(bot_name)) if is_running else mui.IconButton(
                    ^^^^^^^^^^^^^^^^^^
File "/home/dashboard/frontend/components/bot_performance_card.py", line 16, in stop_bot
    backend_api_client.stop_bot(bot_name)
File "/home/dashboard/backend/services/backend_api_client.py", line 123, in stop_bot
    return self.post(endpoint, payload={"bot_name": bot_name, "skip_order_cancellation": skip_order_cancellation,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/dashboard/backend/services/backend_api_client.py", line 40, in post
    return self._process_response(response)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/dashboard/backend/services/backend_api_client.py", line 60, in _process_response
    return response.json()
           ^^^^^^^^^^^^^^^
File "/opt/conda/envs/dashboard/lib/python3.12/site-packages/requests/models.py", line 978, in json
    raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)

Backend-API logs:

INFO:     172.21.0.3:45100 - "GET /get-bot-status/hummingbot-pmm-okx-xrpusdt1-2025.04.18_13.08 HTTP/1.1" 200 OK
INFO:     172.21.0.3:45108 - "POST /stop-bot HTTP/1.1" 500 Internal Server Error
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
    return await self.app(scope, receive, send)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/fastapi/applications.py", line 1054, in __call__
    await super().__call__(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/applications.py", line 112, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/middleware/errors.py", line 187, in __call__
    raise exc
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/middleware/errors.py", line 165, in __call__
    await self.app(scope, receive, _send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/routing.py", line 714, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/routing.py", line 734, in app
    await route.handle(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/routing.py", line 288, in handle
    await self.app(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/routing.py", line 76, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/routing.py", line 73, in app
    response = await f(request)
               ^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/fastapi/routing.py", line 301, in app
    raw_response = await run_endpoint_function(
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/fastapi/routing.py", line 214, in run_endpoint_function
    return await run_in_threadpool(dependant.call, **values)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/starlette/concurrency.py", line 37, in run_in_threadpool
    return await anyio.to_thread.run_sync(func)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/anyio/to_thread.py", line 56, in run_sync
    return await get_async_backend().run_sync_in_worker_thread(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 2470, in run_sync_in_worker_thread
    return await future
           ^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 967, in run
    result = context.run(func, *args)
             ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/backend-api/routers/manage_broker_messages.py", line 56, in stop_bot
    response = bots_manager.stop_bot(action.bot_name, skip_order_cancellation=action.skip_order_cancellation,
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/backend-api/services/bots_orchestrator.py", line 121, in stop_bot
    self.active_bots[bot_name]["broker_listener"].stop()
  File "/backend-api/services/bots_orchestrator.py", line 49, in stop
    super().stop()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/node.py", line 360, in stop
    c.stop()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/pubsub.py", line 146, in stop
    super().stop()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/endpoints.py", line 110, in stop
    self._transport.stop()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/transports/mqtt.py", line 345, in stop
    self.disconnect()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/transports/mqtt.py", line 331, in disconnect
    self._client.disconnect()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 1358, in disconnect
    return self._send_disconnect(reasoncode, properties)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 2853, in _send_disconnect
    return self._packet_queue(command, packet, 0, 0)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 3016, in _packet_queue
    return self.loop_write()
           ^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 1577, in loop_write
    rc = self._packet_write()
         ^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 2504, in _packet_write
    self._do_on_disconnect(MQTT_ERR_SUCCESS)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 3475, in _do_on_disconnect
    on_disconnect(self, self._userdata, rc)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/transports/mqtt.py", line 249, in on_disconnect
    raise ConnectionError()
ConnectionError
Exception in thread Thread-5 (run_forever):
Traceback (most recent call last):
  File "/opt/conda/envs/backend-api/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
    self.run()
  File "/opt/conda/envs/backend-api/lib/python3.12/threading.py", line 1012, in run
    self._target(*self._args, **self._kwargs)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/transports/mqtt.py", line 452, in run_forever
    self._transport.loop_forever()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/transports/mqtt.py", line 353, in loop_forever
    self._client.loop_forever()
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 1756, in loop_forever
    rc = self._loop(timeout)
         ^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 1164, in _loop
    rc = self.loop_read()
         ^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 1558, in loop_read
    return self._loop_rc_handle(rc)
           ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 2350, in _loop_rc_handle
    self._do_on_disconnect(rc, properties)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/paho/mqtt/client.py", line 3475, in _do_on_disconnect
    on_disconnect(self, self._userdata, rc)
  File "/opt/conda/envs/backend-api/lib/python3.12/site-packages/commlib/transports/mqtt.py", line 249, in on_disconnect
    raise ConnectionError()
ConnectionError

Recording: https://www.loom.com/share/fca30a05001945c9881cffc324fed7d1

Steps to reproduce bug

Start hummingbot deploy
Setup any controller-config then deploy instance
Run for couple of minutes then pause
Click stop button and observe loading eventually connectionError

Apr 18 '25 13:04 rapcmia