celery
CRITICAL/MainProcess] Unrecoverable error: OperationalError(2013, 'Lost connection to MySQL server during query')
My celery worker startup command is: celery -A archery worker --loglevel=info. After the service is started, I will see a mysql thread in sleep state on the backend. If I execute a task at this time, the backend will see 5 threads in sleep state. However, because mysql has a timeout, these threads will be closed after 600s. My celery configuration is: from future import absolute_import, unicode_literals import os from celery import Celery #Load configuration os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archery.settings') #Create celery app app = Celery('archery') app.config_from_object('django.conf:settings', namespace='CELERY') #Automatically discover tasks in the project app.autodiscover_tasks()
#Scheduled task app.conf.beat_schedule = {}
My setting configuration: CELERY_RESULT_BACKEND = REDIS_BASE_URL + '1' # Use Redis database 1 to store task results CELERY_RESULT_BACKEND = 'django-db' CELERY_BROKER_URL = REDIS_BASE_URL + '2' # Use Redis database 2 as the message broker CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' CELERYD_CONCURRENCY = 4 CELERY_TIMEZONE = 'Asia/Shanghai' CELERYD_MAX_TASKS_PER_CHILD = 10 CELERYD_FORCE_EXECV = True
Set the default to not save results
CELERY_IGNORE_RESULT = True
CELERY_CREATE_MISSING_QUEUES = True CELERY_DISABLE_RATE_LIMITS = True CELERYD_TASK_SOFT_TIME_LIMIT = 600
CELERY_TASK_RESULT_EXPIRES = 600 CELERY_ENABLE_UTC = False CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json'
My celery worker will burst out the following error after running for a while [2024-09-05 13:28:15,991: CRITICAL/MainProcess] Unrecoverable error: OperationalError(2013, 'Lost connection to MySQL server during query') Traceback (most recent call last): File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/utils.py", line 89, in _execute return self.cursor.execute(sql, params) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/mysql/base.py", line 75, in execute return self.cursor.execute(query, args) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/MySQLdb/cursors.py", line 179, in execute res = self._query(mogrified_query) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/MySQLdb/cursors.py", line 330, in _query db.query(q) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/MySQLdb/connections.py", line 261, in query _mysql.connection.query(self, query) MySQLdb.OperationalError: (2013, 'Lost connection to MySQL server during query')
The above exception was the direct cause of the following exception:
Traceback (most recent call last): File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/worker.py", line 202, in start self.blueprint.start(self) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/bootsteps.py", line 365, in start return self.obj.start() File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/consumer/consumer.py", line 340, in start blueprint.start(self) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/consumer/consumer.py", line 746, in start c.loop(*c.loop_args()) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/loops.py", line 97, in asynloop next(loop) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/asynchronous/hub.py", line 373, in create_loop cb(*cbargs) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/transport/redis.py", line 1352, in on_readable self.cycle.on_readable(fileno) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/transport/redis.py", line 569, in on_readable chan.handlerstype File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/transport/redis.py", line 979, in _brpop_read self.connection._deliver(loads(bytes_to_str(item)), dest) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/transport/virtual/base.py", line 1017, in _deliver callback(message) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/transport/virtual/base.py", line 639, in _callback return callback(message) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/kombu/messaging.py", line 656, in _receive_callback return on_m(message) if on_m else self.receive(decoded, message) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/consumer/consumer.py", line 685, in on_task_received strategy( File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/strategy.py", line 162, in task_message_handler if (req.expires or req.id in revoked_tasks) and req.revoked(): File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/request.py", line 497, in revoked self._announce_revoked( File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/worker/request.py", line 453, in _announce_revoked self.task.backend.mark_as_revoked( File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/backends/base.py", line 267, in mark_as_revoked self.store_result(task_id, exc, state, File "/data/download/venv4archery_archery/lib/python3.10/site-packages/celery/backends/base.py", line 526, in store_result self._store_result(task_id, result, state, traceback, File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django_celery_results/backends/database.py", line 147, in _store_result self.TaskModel._default_manager.store_result(**task_props) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django_celery_results/managers.py", line 42, in _inner return fun(*args, **kwargs) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django_celery_results/managers.py", line 164, in store_result obj, created = self.using(using).get_or_create(task_id=task_id, File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/models/query.py", line 929, in get_or_create return self.get(**kwargs), False File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/models/query.py", line 646, in get num = len(clone) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/models/query.py", line 376, in len self._fetch_all() File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/models/query.py", line 1867, in _fetch_all self._result_cache = list(self._iterable_class(self)) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/models/query.py", line 87, in iter results = compiler.execute_sql( File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1398, in execute_sql cursor.execute(sql, params) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/utils.py", line 102, in execute return super().execute(sql, params) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/utils.py", line 67, in execute return self._execute_with_wrappers( File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/utils.py", line 80, in _execute_with_wrappers return executor(sql, params, many, context) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/utils.py", line 84, in _execute with self.db.wrap_database_errors: File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/utils.py", line 91, in exit raise dj_exc_value.with_traceback(traceback) from exc_value File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/utils.py", line 89, in _execute return self.cursor.execute(sql, params) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/django/db/backends/mysql/base.py", line 75, in execute return self.cursor.execute(query, args) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/MySQLdb/cursors.py", line 179, in execute res = self._query(mogrified_query) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/MySQLdb/cursors.py", line 330, in _query db.query(q) File "/data/download/venv4archery_archery/lib/python3.10/site-packages/MySQLdb/connections.py", line 261, in query _mysql.connection.query(self, query) django.db.utils.OperationalError: (2013, 'Lost connection to MySQL server during query') [2024-09-05 13:28:17,056: WARNING/MainProcess] Restoring 9 unacknowledged message(s)
