Add durable pending runs and recipe list debugging

- Store pending runs in PostgreSQL for durability across restarts
- Add recovery method for orphaned runs
- Increase Celery result_expires to 7 days
- Add task_reject_on_worker_lost for automatic re-queuing
- Add logging to recipe list to debug filter issues

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-11 20:35:00 +00:00
parent a6dd470623
commit 8ab0f05a7d
4 changed files with 412 additions and 31 deletions

View File

@@ -18,7 +18,7 @@ app = Celery(
)
app.conf.update(
result_expires=3600,
result_expires=86400 * 7, # 7 days - allow time for recovery after restarts
task_serializer='json',
accept_content=['json', 'pickle'], # pickle needed for internal Celery messages
result_serializer='json',
@@ -26,8 +26,10 @@ app.conf.update(
timezone='UTC',
enable_utc=True,
task_track_started=True,
task_acks_late=True,
task_acks_late=True, # Don't ack until task completes - survives worker restart
worker_prefetch_multiplier=1,
task_reject_on_worker_lost=True, # Re-queue if worker dies
task_acks_on_failure_or_timeout=True, # Ack failed tasks so they don't retry forever
)
if __name__ == '__main__':