b3ace22009
Heavy pipeline tasks (ingest, translate, render, rerender) now dispatch to a Cloud Run Job (va-worker) instead of local Celery workers. optical-dev runs only api + lightweight worker (notify/embed) within its 2-CPU budget. - backend/app/tasks/runner.py — Cloud Run Job entrypoint - backend/app/services/cloud_run_dispatch.py — replaces .delay() for heavy tasks - backend/Dockerfile.cloudrun — Cloud Run worker image (ffmpeg included) - docker-compose.optical-dev.yml — 2-CPU safe overrides, disables heavy workers - cloudbuild.yaml — builds va-worker image and updates Cloud Run Job - deploy-dev.sh — uses 3-file compose, builds only api+worker locally - routes_jobs, routes_admin_production, ingest_and_ai, translate_and_synthesize — all dispatch sites updated to use cloud_run_dispatch.dispatch() USE_CELERY_FALLBACK=true in .env.local to use Celery locally during dev. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
87 lines
2.2 KiB
YAML
87 lines
2.2 KiB
YAML
# =============================================================================
|
|
# optical-dev overrides — 2 CPU / ~8 GB RAM server
|
|
#
|
|
# Heavy pipeline workers (ingest, translate, render, rerender) run on
|
|
# Cloud Run Jobs. Only lightweight services run here.
|
|
#
|
|
# Usage:
|
|
# docker compose -f docker-compose.yml \
|
|
# -f docker-compose.prod.yml \
|
|
# -f docker-compose.optical-dev.yml \
|
|
# --env-file .env.production up -d
|
|
# =============================================================================
|
|
|
|
services:
|
|
# ── Keep on this server, resource limits fit in 2 CPU ──────────────────────
|
|
|
|
mongodb:
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 1G
|
|
cpus: '0.5'
|
|
reservations:
|
|
memory: 512M
|
|
cpus: '0.25'
|
|
|
|
redis:
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 512M
|
|
cpus: '0.25'
|
|
reservations:
|
|
memory: 256M
|
|
cpus: '0.1'
|
|
|
|
api:
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 2G
|
|
cpus: '1.0'
|
|
reservations:
|
|
memory: 1G
|
|
cpus: '0.5'
|
|
environment:
|
|
APP_ENV: prod
|
|
# Cloud Run dispatch config
|
|
CLOUD_RUN_WORKER_JOB: va-worker
|
|
GCP_REGION: europe-west1
|
|
USE_CELERY_FALLBACK: "false"
|
|
|
|
# Lightweight worker: only notify + embed_glossary tasks
|
|
# Heavy tasks (ingest/translate/render) go to Cloud Run Jobs
|
|
worker:
|
|
deploy:
|
|
resources:
|
|
limits:
|
|
memory: 512M
|
|
cpus: '0.25'
|
|
reservations:
|
|
memory: 256M
|
|
cpus: '0.1'
|
|
environment:
|
|
APP_ENV: prod
|
|
# Only consume lightweight queues; heavy queues handled by Cloud Run
|
|
CELERY_QUEUES: "notify,embed"
|
|
command: >
|
|
celery -A app.tasks worker
|
|
--loglevel=info
|
|
--queues=notify,embed
|
|
--concurrency=2
|
|
--hostname=lite-worker@%h
|
|
|
|
# ── Disabled on optical-dev — run on Cloud Run Jobs instead ───────────────
|
|
|
|
ffmpeg-worker:
|
|
deploy:
|
|
replicas: 0
|
|
|
|
tts-worker:
|
|
deploy:
|
|
replicas: 0
|
|
|
|
whisper-worker:
|
|
deploy:
|
|
replicas: 0
|