Maximum recursion depth exceeded

[lbatteau]

When launching a Celery task from our FastAPI service, we are intermittently getting the following exception:

File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 99, in wait
    self.connection.drain_events(timeout=timeout)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 525, in drain_events
    while not self.blocking_read(timeout):
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 531, in blocking_read
    return self.on_inbound_frame(frame)
  File "/usr/local/lib/python3.10/site-packages/amqp/method_framing.py", line 53, in on_frame
    callback(channel, method_sig, buf, None)
  File "/usr/local/lib/python3.10/site-packages/amqp/connection.py", line 537, in on_inbound_method
    return self.channels[channel_id].dispatch_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 156, in dispatch_method
    listener(*args)
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 292, in _on_close
    self._do_revive()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 177, in _do_revive
    self.open()
  File "/usr/local/lib/python3.10/site-packages/amqp/channel.py", line 448, in open
    return self.send_method(
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 79, in send_method
    return self.wait(wait, returns_tuple=returns_tuple)
  File "/usr/local/lib/python3.10/site-packages/amqp/abstract_channel.py", line 87, in wait
    p = ensure_promise(callback)
  File "/usr/local/lib/python3.10/site-packages/vine/funtools.py", line 26, in ensure_promise
    return promise()
  File "/usr/local/lib/python3.10/site-packages/vine/promises.py", line 88, in __init__
    self.fun = self._get_fun_or_weakref(fun=fun, weak=weak)
RecursionError: maximum recursion depth exceeded

Haven't ever seen this before, or found anything resembling this error on the internet. We have a setup with a FastAPI service, a separate container running Celery and a RabbitMQ broker. When we restart the service, it works again.

Versions: Celery: 5.3.1 py-amqp: 5.1.1