9580979ac8
Allow configuring Celery worker concurrency via environment variables
to take advantage of Cloud Run autoscaling:
- Add WORKER_CONCURRENCY, WHISPER_WORKER_CONCURRENCY, FFMPEG_WORKER_CONCURRENCY
settings to config.py with recommended values documented
- Update Dockerfile to use ${WORKER_CONCURRENCY} and ${WHISPER_WORKER_CONCURRENCY}
environment variables instead of hardcoded values
- Update docker-compose.yml to pass concurrency env vars to worker commands
- Add WHISPER_SERVICE_URL and FFMPEG_SERVICE_URL to relevant workers
Recommended settings:
Local mode: WHISPER=1, FFMPEG=1 (CPU/RAM constrained)
Cloud Run mode: WHISPER=10, FFMPEG=20 (match autoscaling limits)
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
190 lines
6.7 KiB
Docker
190 lines
6.7 KiB
Docker
# =============================================================================
|
|
# Multi-stage Dockerfile for Accessible Video Processing Platform
|
|
# =============================================================================
|
|
# Stage 1: Builder - Install dependencies
|
|
# Stage 2: Base - Common runtime for API and Worker
|
|
# Stage 3: API - FastAPI + Gunicorn (with ffmpeg for TTS audio conversion)
|
|
# Stage 4: Worker - Celery worker (with ffmpeg for video processing)
|
|
# =============================================================================
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Stage 1: Builder - Install Python dependencies using Poetry
|
|
# -----------------------------------------------------------------------------
|
|
FROM python:3.11-slim AS builder
|
|
|
|
# Install build dependencies
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
curl \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install Poetry
|
|
RUN pip install --no-cache-dir poetry==1.8.2
|
|
|
|
# Configure Poetry to not create virtual environment (we're in a container)
|
|
ENV POETRY_NO_INTERACTION=1 \
|
|
POETRY_VIRTUALENVS_CREATE=false \
|
|
POETRY_CACHE_DIR=/tmp/poetry_cache
|
|
|
|
WORKDIR /app
|
|
|
|
# Copy dependency files
|
|
COPY pyproject.toml poetry.lock ./
|
|
|
|
# Install dependencies using Poetry directly (simpler and more reliable)
|
|
RUN poetry config virtualenvs.create false \
|
|
&& poetry install --only main --no-interaction --no-ansi \
|
|
&& rm -rf $POETRY_CACHE_DIR
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Stage 2: Base - Common runtime environment
|
|
# -----------------------------------------------------------------------------
|
|
FROM python:3.11-slim AS base
|
|
|
|
# Install common runtime dependencies
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
libmagic1 \
|
|
curl \
|
|
tini \
|
|
&& rm -rf /var/lib/apt/lists/* \
|
|
&& apt-get clean
|
|
|
|
# Create non-root user for security
|
|
RUN groupadd --gid 1000 app \
|
|
&& useradd --uid 1000 --gid app --shell /bin/bash --create-home app
|
|
|
|
# Copy Python packages from builder (installed globally, not in user dir)
|
|
COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
|
|
COPY --from=builder /usr/local/bin /usr/local/bin
|
|
|
|
# Set environment variables
|
|
ENV PYTHONPATH=/app \
|
|
PYTHONUNBUFFERED=1 \
|
|
PYTHONDONTWRITEBYTECODE=1
|
|
|
|
WORKDIR /app
|
|
|
|
# Copy application code
|
|
COPY --chown=app:app . .
|
|
|
|
# Switch to non-root user
|
|
USER app
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Stage 3: API - FastAPI + Gunicorn (Production API Server)
|
|
# -----------------------------------------------------------------------------
|
|
FROM base AS api
|
|
|
|
# Switch to root to install ffmpeg
|
|
USER root
|
|
|
|
# Install ffmpeg for TTS audio conversion
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ffmpeg \
|
|
&& rm -rf /var/lib/apt/lists/* \
|
|
&& apt-get clean
|
|
|
|
# Switch back to non-root user
|
|
USER app
|
|
|
|
# Set production environment variables
|
|
ENV APP_ENV=prod
|
|
|
|
# Health check for API
|
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
|
CMD curl -f http://localhost:8000/health || exit 1
|
|
|
|
# Expose API port
|
|
EXPOSE 8000
|
|
|
|
# Use tini as init system for proper signal handling
|
|
ENTRYPOINT ["tini", "--"]
|
|
|
|
# Start Gunicorn with Uvicorn workers
|
|
CMD ["gunicorn", "-c", "gunicorn_conf.py", "app.main:app"]
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Stage 4: Worker - Celery Worker (with ffmpeg for video processing)
|
|
# -----------------------------------------------------------------------------
|
|
FROM base AS worker
|
|
|
|
# Switch back to root to install ffmpeg
|
|
USER root
|
|
|
|
# Install ffmpeg for video processing
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ffmpeg \
|
|
&& rm -rf /var/lib/apt/lists/* \
|
|
&& apt-get clean
|
|
|
|
# Switch back to non-root user
|
|
USER app
|
|
|
|
# Set production environment variables
|
|
# WORKER_CONCURRENCY can be overridden at runtime (default: 8)
|
|
ENV APP_ENV=prod \
|
|
C_FORCE_ROOT=0 \
|
|
WORKER_CONCURRENCY=8
|
|
|
|
# Health check for worker (check if Celery is responding)
|
|
HEALTHCHECK --interval=60s --timeout=15s --start-period=10s --retries=3 \
|
|
CMD python -c "from celery import Celery; app=Celery('accessible-video-tasks', broker='redis://redis:6379/0'); app.control.inspect().ping() or exit(1)" || exit 1
|
|
|
|
# Use tini as init system for proper signal handling
|
|
ENTRYPOINT ["tini", "--"]
|
|
|
|
# Start Celery worker listening to all queues EXCEPT whisper
|
|
# Whisper has a dedicated worker (see Stage 5) to prevent memory overload
|
|
# Concurrency is set via WORKER_CONCURRENCY env var (default: 8)
|
|
# When using Cloud Run for FFmpeg, can increase to handle more parallel HTTP calls
|
|
CMD celery -A celery_worker worker \
|
|
-Q default,ingest,notify,render,ffmpeg \
|
|
--loglevel=info \
|
|
--concurrency=${WORKER_CONCURRENCY} \
|
|
--max-tasks-per-child=100
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Stage 5: Whisper Worker - Dedicated worker for Whisper transcription
|
|
# -----------------------------------------------------------------------------
|
|
FROM base AS whisper-worker
|
|
|
|
# Switch back to root to install ffmpeg
|
|
USER root
|
|
|
|
# Install ffmpeg for audio extraction
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ffmpeg \
|
|
&& rm -rf /var/lib/apt/lists/* \
|
|
&& apt-get clean
|
|
|
|
# Switch back to non-root user
|
|
USER app
|
|
|
|
# Pre-download Whisper medium model during build to avoid cold start delays
|
|
# Model is cached in ~/.cache/huggingface/hub (~1.5GB)
|
|
RUN python -c "from faster_whisper import WhisperModel; WhisperModel('medium', device='cpu', compute_type='int8')"
|
|
|
|
# Set production environment variables
|
|
# WHISPER_WORKER_CONCURRENCY can be overridden at runtime
|
|
# Default: 1 for local mode (RAM constrained), set to 10 for Cloud Run mode
|
|
ENV APP_ENV=prod \
|
|
C_FORCE_ROOT=0 \
|
|
WHISPER_WORKER_CONCURRENCY=1
|
|
|
|
# Health check for worker (check if Celery is responding)
|
|
HEALTHCHECK --interval=60s --timeout=15s --start-period=10s --retries=3 \
|
|
CMD python -c "from celery import Celery; app=Celery('accessible-video-tasks', broker='redis://redis:6379/0'); app.control.inspect().ping() or exit(1)" || exit 1
|
|
|
|
# Use tini as init system for proper signal handling
|
|
ENTRYPOINT ["tini", "--"]
|
|
|
|
# Start Celery worker listening ONLY to whisper queue
|
|
# Concurrency is set via WHISPER_WORKER_CONCURRENCY env var
|
|
# Local mode: 1 (RAM constrained - Whisper model uses ~4-6GB)
|
|
# Cloud Run mode: 10 (just HTTP calls, match Cloud Run max instances)
|
|
# --max-tasks-per-child=50 to periodically recycle workers and free memory
|
|
CMD celery -A celery_worker worker \
|
|
-Q whisper \
|
|
--loglevel=info \
|
|
--concurrency=${WHISPER_WORKER_CONCURRENCY} \
|
|
--max-tasks-per-child=50
|