b3ace22009
Heavy pipeline tasks (ingest, translate, render, rerender) now dispatch to a Cloud Run Job (va-worker) instead of local Celery workers. optical-dev runs only api + lightweight worker (notify/embed) within its 2-CPU budget. - backend/app/tasks/runner.py — Cloud Run Job entrypoint - backend/app/services/cloud_run_dispatch.py — replaces .delay() for heavy tasks - backend/Dockerfile.cloudrun — Cloud Run worker image (ffmpeg included) - docker-compose.optical-dev.yml — 2-CPU safe overrides, disables heavy workers - cloudbuild.yaml — builds va-worker image and updates Cloud Run Job - deploy-dev.sh — uses 3-file compose, builds only api+worker locally - routes_jobs, routes_admin_production, ingest_and_ai, translate_and_synthesize — all dispatch sites updated to use cloud_run_dispatch.dispatch() USE_CELERY_FALLBACK=true in .env.local to use Celery locally during dev. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
57 lines
2 KiB
YAML
57 lines
2 KiB
YAML
# =============================================================================
|
|
# Cloud Build — build va-worker image and push to Artifact Registry
|
|
#
|
|
# Trigger: manual or on push to main
|
|
# Usage:
|
|
# gcloud builds submit --config cloudbuild.yaml .
|
|
# =============================================================================
|
|
|
|
substitutions:
|
|
_REGION: europe-west1
|
|
_REPO: nexus
|
|
_IMAGE: va-worker
|
|
_TAG: $COMMIT_SHA # replaced with actual SHA by Cloud Build; use "latest" for manual runs
|
|
|
|
steps:
|
|
# ── Build Cloud Run worker image ──────────────────────────────────────────
|
|
- name: gcr.io/cloud-builders/docker
|
|
id: build-va-worker
|
|
args:
|
|
- build
|
|
- -f
|
|
- backend/Dockerfile.cloudrun
|
|
- -t
|
|
- ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_IMAGE}:${_TAG}
|
|
- -t
|
|
- ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_IMAGE}:latest
|
|
- backend/
|
|
|
|
# ── Push both tags ────────────────────────────────────────────────────────
|
|
- name: gcr.io/cloud-builders/docker
|
|
id: push-va-worker
|
|
args:
|
|
- push
|
|
- --all-tags
|
|
- ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_IMAGE}
|
|
|
|
# ── Update Cloud Run Job to use new image ─────────────────────────────────
|
|
- name: gcr.io/google.com/cloudsdktool/cloud-sdk
|
|
id: update-cloud-run-job
|
|
entrypoint: gcloud
|
|
args:
|
|
- run
|
|
- jobs
|
|
- update
|
|
- va-worker
|
|
- --image
|
|
- ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_IMAGE}:${_TAG}
|
|
- --region
|
|
- ${_REGION}
|
|
|
|
images:
|
|
- ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_IMAGE}:${_TAG}
|
|
- ${_REGION}-docker.pkg.dev/$PROJECT_ID/${_REPO}/${_IMAGE}:latest
|
|
|
|
options:
|
|
logging: CLOUD_LOGGING_ONLY
|
|
machineType: E2_HIGHCPU_8 # faster builds
|