[Bug]: [Python] Redis disconnect sends into error loop and hangs app

[RolandasRazma]

Version

2.11.0

Platform

Python

What happened?

Redis disconnect send python app into error loop hanging app

How to reproduce.

stop redis

async def processor(job: Job, job_token: str):
    pass


async def main():
    # Shutdown event
    shutdown_event = asyncio.Event()

    def signal_handler(signal, frame):
        logger.info(f"got signal {signal}")
        shutdown_event.set()

    # Assign signal handlers to SIGTERM and SIGINT
    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGINT, signal_handler)

    worker = Worker(name="queue_name_1", processor=processor)

    await shutdown_event.wait()
    # close the worker
    await worker.close()


if __name__ == "__main__":
    asyncio.run(main())

Relevant log output

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/xxx/lib/python3.12/site-packages/bullmq/utils.py", line 11, in extract_result
    return job_task.result()
           ^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/bullmq/worker.py", line 122, in getNextJob
    job_instance = await self.moveToActive(token)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/bullmq/worker.py", line 126, in moveToActive
    result = await self.scripts.moveToActive(token, self.opts)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/bullmq/scripts.py", line 478, in moveToActive
    result = await self.commands["moveToActive"](keys=keys, args=args)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/commands/core.py", line 5572, in __call__
    return await client.evalsha(self.sha, len(keys), *args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/client.py", line 611, in execute_command
    conn = self.connection or await pool.get_connection(command_name, **options)
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 1064, in get_connection
    await self.ensure_connection(connection)
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 1097, in ensure_connection
    await connection.connect()
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 289, in connect
    raise ConnectionError(self._error_message(e))
redis.exceptions.ConnectionError: Error Multiple exceptions: [Errno 61] Connect call failed ('127.0.0.1', 6379), [Errno 61] Connect call failed ('::1', 6379, 0, 0) connecting to localhost:6379.
Traceback (most recent call last):
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 281, in connect
    await self.retry.call_with_retry(
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/retry.py", line 59, in call_with_retry
    return await do()
           ^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 697, in _connect
    reader, writer = await asyncio.open_connection(
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/asyncio/streams.py", line 48, in open_connection
    transport, _ = await loop.create_connection(
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/asyncio/base_events.py", line 1140, in create_connection
    raise OSError('Multiple exceptions: {}'.format(
OSError: Multiple exceptions: [Errno 61] Connect call failed ('127.0.0.1', 6379), [Errno 61] Connect call failed ('::1', 6379, 0, 0)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/xxx/lib/python3.12/site-packages/bullmq/utils.py", line 11, in extract_result
    return job_task.result()
           ^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/bullmq/worker.py", line 122, in getNextJob
    job_instance = await self.moveToActive(token)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/bullmq/worker.py", line 126, in moveToActive
    result = await self.scripts.moveToActive(token, self.opts)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/bullmq/scripts.py", line 478, in moveToActive
    result = await self.commands["moveToActive"](keys=keys, args=args)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/commands/core.py", line 5572, in __call__
    return await client.evalsha(self.sha, len(keys), *args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/client.py", line 611, in execute_command
    conn = self.connection or await pool.get_connection(command_name, **options)
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 1064, in get_connection
    await self.ensure_connection(connection)
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 1097, in ensure_connection
    await connection.connect()
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 289, in connect
    raise ConnectionError(self._error_message(e))
redis.exceptions.ConnectionError: Error Multiple exceptions: [Errno 61] Connect call failed ('127.0.0.1', 6379), [Errno 61] Connect call failed ('::1', 6379, 0, 0) connecting to localhost:6379.
Traceback (most recent call last):
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 281, in connect
    await self.retry.call_with_retry(
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/retry.py", line 59, in call_with_retry
    return await do()
           ^^^^^^^^^^
  File "/xxx/lib/python3.12/site-packages/redis/asyncio/connection.py", line 697, in _connect
    reader, writer = await asyncio.open_connection(
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/asyncio/streams.py", line 48, in open_connection
    transport, _ = await loop.create_connection(
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/xxx/lib/python3.12/asyncio/base_events.py", line 1140, in create_connection
    raise OSError('Multiple exceptions: {}'.format(
OSError: Multiple exceptions: [Errno 61] Connect call failed ('127.0.0.1', 6379), [Errno 61] Connect call failed ('::1', 6379, 0, 0)

During handling of the above exception, another exception occurred:

Code of Conduct

• [x] I agree to follow this project's Code of Conduct

[RolandasRazma]

Actually I see same behavior no matter processor(job: Job, job_token: str):. Just start your sample code without Redis alive (updated report)