cloud-reporter/backend/app/celery_app.py

import os
from celery import Celery

# Create Celery application
celery_app = Celery(
    'aws_scanner',
    broker=os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),
    backend=os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/1'),
    include=['app.tasks.scan_tasks']
)

# Celery configuration
celery_app.conf.update(
    # Serialization
    task_serializer='json',
    accept_content=['json'],
    result_serializer='json',
    
    # Timezone
    timezone='UTC',
    enable_utc=True,
    
    # Task tracking
    task_track_started=True,
    
    # Timeouts
    task_time_limit=3600,  # 1 hour hard limit
    task_soft_time_limit=3300,  # 55 minutes soft limit
    
    # Worker settings
    worker_prefetch_multiplier=1,  # Each worker takes one task at a time
    task_acks_late=True,  # Acknowledge task after completion
    
    # Result settings
    result_expires=86400,  # Results expire after 24 hours
    
    # Retry settings
    task_default_retry_delay=60,  # 60 seconds default retry delay
    task_max_retries=3,  # Maximum 3 retries
    
    # Broker connection retry settings (Celery 6.0+ compatibility)
    broker_connection_retry_on_startup=True,
    
    # Beat schedule (for periodic tasks if needed)
    beat_schedule={
        'cleanup-old-reports': {
            'task': 'app.tasks.scan_tasks.cleanup_old_reports',
            'schedule': 86400.0,  # Run daily
            'args': (30,)  # Keep reports for 30 days
        },
    },
)


def init_celery(app):
    """Initialize Celery with Flask application context"""
    celery_app.conf.update(
        broker_url=app.config.get('CELERY_BROKER_URL', 'redis://localhost:6379/0'),
        result_backend=app.config.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379/1'),
    )
    
    class ContextTask(celery_app.Task):
        """Task that runs within Flask application context"""
        def __call__(self, *args, **kwargs):
            with app.app_context():
                return self.run(*args, **kwargs)
    
    celery_app.Task = ContextTask
    return celery_app