diff --git a/.env.prod.example b/.env.prod.example new file mode 100644 index 0000000..ebf52d9 --- /dev/null +++ b/.env.prod.example @@ -0,0 +1,34 @@ +# Production Environment Variables +# Copy this file to .env.prod and update with your production values + +# Database Configuration +MONGODB_ROOT_USER=admin +MONGODB_ROOT_PASSWORD=your-secure-mongodb-password +MONGODB_URL=mongodb://admin:your-secure-mongodb-password@mongodb:27017/accessible_video?authSource=admin&replicaSet=rs0 +REDIS_URL=redis://redis:6379/0 + +# JWT Authentication +JWT_SECRET_KEY=your-production-jwt-secret-key-min-32-chars +JWT_REFRESH_SECRET_KEY=your-production-refresh-secret-key-min-32-chars + +# AI Services +GEMINI_API_KEY=your-gemini-api-key +ELEVENLABS_API_KEY=your-elevenlabs-api-key + +# Google Cloud Configuration +GCS_BUCKET_NAME=your-production-bucket-name +GOOGLE_CLOUD_PROJECT=your-gcp-project-id + +# Email Service +SENDGRID_API_KEY=your-sendgrid-api-key + +# Monitoring +SENTRY_DSN=your-sentry-dsn-url + +# CORS Configuration for Apache-hosted frontend +CORS_ORIGINS=https://your-domain.com,https://www.your-domain.com + +# Frontend Build Configuration (for reference) +VITE_API_URL=https://your-api-domain.com:8000 +VITE_SENTRY_DSN=your-frontend-sentry-dsn +VITE_ENVIRONMENT=production \ No newline at end of file diff --git a/APACHE_DEPLOYMENT.md b/APACHE_DEPLOYMENT.md new file mode 100644 index 0000000..fc852a2 --- /dev/null +++ b/APACHE_DEPLOYMENT.md @@ -0,0 +1,236 @@ +# Apache Frontend + Docker Backend Deployment Guide + +## 🏗 Architecture Overview + +**Frontend**: Built React app served by your existing Apache webserver +**Backend**: Docker containers running FastAPI + workers + database + +``` +Apache Webserver (Frontend) → Docker Backend Services + └── Built React App ├── FastAPI API (:8000) + ├── Celery Workers + ├── Change Stream Service + ├── MongoDB + └── Redis +``` + +## 🚀 Deployment Steps + +### 1. **Deploy Backend Services** + +```bash +# 1. Create production environment file +cp .env.prod.example .env.prod +# Edit .env.prod with your production values + +# 2. Start backend services only +docker-compose -f docker-compose.prod.yml up -d + +# 3. Verify services are running +docker-compose -f docker-compose.prod.yml ps +``` + +**Running Services:** +- `accessible-video-api-prod` - FastAPI API (port 8000) +- `accessible-video-worker-prod` - Celery workers +- `accessible-video-mongo-prod` - MongoDB database +- `accessible-video-redis-prod` - Redis cache/queue + +### 2. **Build and Deploy Frontend to Apache** + +```bash +# 1. Configure frontend environment +cd frontend +cp .env.example .env.production.local + +# Edit .env.production.local: +# VITE_API_URL=https://your-api-domain.com:8000 +# VITE_SENTRY_DSN=your-sentry-dsn +# VITE_ENVIRONMENT=production + +# 2. Build production frontend +npm run build + +# 3. Deploy to Apache document root +sudo cp -r dist/* /var/www/html/your-app/ +# OR +sudo rsync -av --delete dist/ /var/www/html/your-app/ +``` + +### 3. **Configure Apache Virtual Host** + +Create `/etc/apache2/sites-available/your-app.conf`: + +```apache + + ServerName your-domain.com + ServerAlias www.your-domain.com + DocumentRoot /var/www/html/your-app + + # SSL Configuration + SSLEngine on + SSLCertificateFile /path/to/your/certificate.crt + SSLCertificateKeyFile /path/to/your/private.key + + # Security Headers + Header always set X-Frame-Options "SAMEORIGIN" + Header always set X-Content-Type-Options "nosniff" + Header always set X-XSS-Protection "1; mode=block" + Header always set Referrer-Policy "strict-origin-when-cross-origin" + Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains" + + # Compression + + AddOutputFilterByType DEFLATE text/plain + AddOutputFilterByType DEFLATE text/html + AddOutputFilterByType DEFLATE text/xml + AddOutputFilterByType DEFLATE text/css + AddOutputFilterByType DEFLATE application/xml + AddOutputFilterByType DEFLATE application/xhtml+xml + AddOutputFilterByType DEFLATE application/rss+xml + AddOutputFilterByType DEFLATE application/javascript + AddOutputFilterByType DEFLATE application/x-javascript + + + # Caching for static assets + + ExpiresActive On + ExpiresDefault "access plus 1 year" + Header set Cache-Control "public, immutable" + + + # Don't cache HTML files + + ExpiresActive On + ExpiresDefault "access plus 0 seconds" + Header set Cache-Control "no-cache, no-store, must-revalidate" + + + # React Router support (handle client-side routing) + + Options -Indexes + AllowOverride All + Require all granted + + # Fallback to index.html for client-side routing + FallbackResource /index.html + + + # Optional: Proxy API requests (alternative to CORS) + # ProxyPreserveHost On + # ProxyPass /api/ http://your-docker-host:8000/api/ + # ProxyPassReverse /api/ http://your-docker-host:8000/api/ + + # Logs + ErrorLog ${APACHE_LOG_DIR}/your-app_error.log + CustomLog ${APACHE_LOG_DIR}/your-app_access.log combined + + +# HTTP to HTTPS redirect + + ServerName your-domain.com + ServerAlias www.your-domain.com + Redirect permanent / https://your-domain.com/ + +``` + +Enable the site: +```bash +sudo a2ensite your-app.conf +sudo systemctl reload apache2 +``` + +## ⚙️ Configuration Files Updated + +### `docker-compose.prod.yml` +- ✅ Removed frontend and nginx services +- ✅ Added CORS_ORIGINS environment variable +- ✅ Backend services only (API, workers, database) + +### `.env.prod.example` +- ✅ Production environment template +- ✅ CORS configuration for Apache frontend +- ✅ All required variables documented + +## 🔧 CORS Configuration + +Since frontend and backend are on different domains, configure CORS in your backend: + +**In `.env.prod`:** +```bash +CORS_ORIGINS=https://your-domain.com,https://www.your-domain.com +``` + +**Backend automatically handles CORS** based on this environment variable. + +## 📋 Deployment Checklist + +### Backend Services +- [ ] Copy `.env.prod.example` to `.env.prod` +- [ ] Update all environment variables in `.env.prod` +- [ ] Run `docker-compose -f docker-compose.prod.yml up -d` +- [ ] Verify API accessible at `http://your-docker-host:8000/docs` +- [ ] Check logs: `docker-compose -f docker-compose.prod.yml logs -f` + +### Frontend Deployment +- [ ] Update `frontend/.env.production.local` with API URL +- [ ] Run `npm run build` in frontend directory +- [ ] Copy `dist/*` to Apache document root +- [ ] Configure Apache virtual host +- [ ] Enable site and reload Apache +- [ ] Test frontend loads and connects to API + +### Security & Performance +- [ ] SSL certificate configured +- [ ] Security headers enabled +- [ ] Gzip compression enabled +- [ ] Static file caching configured +- [ ] CORS origins properly set +- [ ] Firewall rules: only expose port 8000 for API + +## 🔍 Troubleshooting + +### Common Issues + +**CORS Errors:** +- Verify `CORS_ORIGINS` in `.env.prod` matches your domain +- Check browser dev tools for exact error + +**API Connection Failed:** +- Verify `VITE_API_URL` in frontend build +- Check backend API is accessible from frontend server +- Ensure port 8000 is open and reachable + +**React Router 404s:** +- Verify `FallbackResource /index.html` in Apache config +- Ensure `AllowOverride All` is set + +**File Upload Issues:** +- Check Apache `LimitRequestBody` directive +- Verify backend can write to GCS bucket + +### Monitoring Commands + +```bash +# Backend services status +docker-compose -f docker-compose.prod.yml ps + +# View logs +docker-compose -f docker-compose.prod.yml logs -f api +docker-compose -f docker-compose.prod.yml logs -f worker + +# Apache status +sudo systemctl status apache2 +sudo tail -f /var/log/apache2/your-app_error.log +``` + +## 🎯 Benefits of This Setup + +✅ **Separation of Concerns** - Frontend and backend independently deployable +✅ **Existing Infrastructure** - Uses your current Apache setup +✅ **Scalability** - Backend can be moved to different hosts easily +✅ **Caching** - Apache handles static file caching efficiently +✅ **SSL Termination** - Apache handles HTTPS for frontend +✅ **Monitoring** - Separate logs and monitoring for each tier + +Your backend services will run in Docker containers while the frontend integrates seamlessly with your existing Apache web server infrastructure. \ No newline at end of file diff --git a/DEPLOYMENT_OPTIONS.md b/DEPLOYMENT_OPTIONS.md new file mode 100644 index 0000000..50663cf --- /dev/null +++ b/DEPLOYMENT_OPTIONS.md @@ -0,0 +1,168 @@ +# Deployment Options for Video Accessibility Platform + +## 🏗 Current Docker Setup + +Your `docker-compose.yml` serves **both frontend and backend** in **development mode**: + +- **Frontend**: Vite dev server on port 5173 (hot reload) +- **Backend**: FastAPI on port 8000 (auto-reload) +- **Database**: MongoDB + Redis +- **Workers**: Celery + Change Stream service + +## 🚀 Production Deployment Options + +### 1. **All-in-Docker Production** ✅ Recommended + +**What it does:** +- Frontend: Built React app served by Nginx (port 80) +- Backend: Production FastAPI (port 8000) +- Single `docker-compose up` deployment + +**Usage:** +```bash +# Production deployment +docker-compose -f docker-compose.prod.yml up -d + +# Access: +# Frontend: http://localhost:80 +# Backend API: http://localhost:8000 +``` + +**Benefits:** +- ✅ Single command deployment +- ✅ Optimized frontend build +- ✅ Production-ready configuration +- ✅ Built-in health checks +- ✅ Nginx caching and compression + +### 2. **Single Domain with Nginx Proxy** ✅ Best UX + +**What it does:** +- Everything served from one domain (port 80) +- `/api/*` routes to backend +- `/*` routes to frontend +- WebSocket support included + +**Usage:** +```bash +# Uses nginx/nginx.conf for routing +docker-compose -f docker-compose.prod.yml up nginx + +# Access everything at: http://localhost +``` + +**Benefits:** +- ✅ No CORS issues +- ✅ Single domain simplicity +- ✅ Better caching control +- ✅ Rate limiting built-in +- ✅ SSL termination ready + +### 3. **Cloud-Native (Google Cloud)** 🌟 Enterprise + +**Architecture:** +``` +Frontend (Cloud Storage + CDN) → API (Cloud Run) → Database (MongoDB Atlas) + ↓ + Workers (Cloud Run) +``` + +**Components:** +- **Frontend**: Build + deploy to Cloud Storage, serve via Cloud CDN +- **Backend**: Deploy to Cloud Run (auto-scaling) +- **Workers**: Separate Cloud Run service for Celery +- **Database**: MongoDB Atlas (managed) +- **Files**: Google Cloud Storage (already integrated) + +**Benefits:** +- ✅ Auto-scaling +- ✅ Global CDN +- ✅ Managed services +- ✅ Pay-per-use +- ✅ High availability + +## 📊 Comparison Matrix + +| Option | Complexity | Cost | Scalability | Maintenance | +|--------|------------|------|-------------|-------------| +| **Dev Docker** | Low | Very Low | Limited | Manual | +| **Prod Docker** | Low | Low | Manual | Medium | +| **Nginx Proxy** | Medium | Low | Manual | Medium | +| **Cloud Native** | High | Variable | Automatic | Low | + +## 🚀 Quick Migration Guide + +### From Development → Production Docker + +1. **Update environment variables:** + ```bash + cp .env.example .env.prod + # Edit .env.prod with production values + ``` + +2. **Deploy:** + ```bash + docker-compose -f docker-compose.prod.yml up -d + ``` + +3. **Verify:** + ```bash + # Frontend (optimized build) + curl http://localhost:80 + + # Backend API + curl http://localhost:8000/health + ``` + +### From Docker → Cloud Native + +1. **Build frontend:** + ```bash + cd frontend && npm run build + gsutil -m rsync -r -d dist/ gs://your-bucket/ + ``` + +2. **Deploy backend:** + ```bash + gcloud run deploy video-api --source=./backend --region=us-central1 + ``` + +3. **Deploy workers:** + ```bash + gcloud run deploy video-workers --source=./backend --region=us-central1 + ``` + +## 🔧 Configuration Files Created + +### `docker-compose.prod.yml` +- Production-ready Docker setup +- Nginx serving frontend +- Optimized environment variables +- Health checks included + +### `nginx/nginx.conf` +- Single-domain routing configuration +- API proxy with rate limiting +- WebSocket support +- Static file caching +- Security headers + +## 🎯 Recommendations by Use Case + +### **Small Team / MVP** +→ Use **Production Docker** (`docker-compose.prod.yml`) + +### **Growing Business** +→ Use **Nginx Proxy** setup for better performance + +### **Enterprise / Scale** +→ Go **Cloud Native** with Google Cloud Run + CDN + +## 🔍 Current Status + +✅ **Development**: Already working with `docker-compose up` +✅ **Production Docker**: Ready with `docker-compose.prod.yml` +✅ **Nginx Proxy**: Configured and ready to deploy +⚠️ **Cloud Native**: Requires GCP setup and configuration + +Your current Docker setup is **development-optimized**. For production, use the new `docker-compose.prod.yml` which properly builds and serves the React app through Nginx while keeping the backend API separate but coordinated. \ No newline at end of file diff --git a/Makefile b/Makefile index 7f18d4f..c43d466 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ dev-frontend: ## Start frontend development server cd frontend && npm run dev dev-worker: ## Start Celery worker - cd backend && poetry run celery -A celery_worker.celery_app worker --loglevel=info + cd backend && poetry run celery -A app.tasks worker --loglevel=info + test-backend: ## Run backend tests cd backend && poetry run pytest diff --git a/backend/app/api/v1/__pycache__/routes_admin.cpython-313.pyc b/backend/app/api/v1/__pycache__/routes_admin.cpython-313.pyc index 81978ae..d5dbc3d 100644 Binary files a/backend/app/api/v1/__pycache__/routes_admin.cpython-313.pyc and b/backend/app/api/v1/__pycache__/routes_admin.cpython-313.pyc differ diff --git a/backend/app/api/v1/__pycache__/routes_jobs.cpython-313.pyc b/backend/app/api/v1/__pycache__/routes_jobs.cpython-313.pyc index da8c9f4..2f1b8fe 100644 Binary files a/backend/app/api/v1/__pycache__/routes_jobs.cpython-313.pyc and b/backend/app/api/v1/__pycache__/routes_jobs.cpython-313.pyc differ diff --git a/backend/app/api/v1/routes_admin.py b/backend/app/api/v1/routes_admin.py index db80287..e6c6317 100644 --- a/backend/app/api/v1/routes_admin.py +++ b/backend/app/api/v1/routes_admin.py @@ -599,6 +599,18 @@ async def reprocess_job( } ) + # Broadcast status update + try: + from ...services.websocket import connection_manager + await connection_manager.broadcast_job_status_update( + job_id=job_id, + status="created", + job_title=job_doc.get("title"), + message=f"{job_doc.get('title', 'Job')} has been reset and is queued for reprocessing" + ) + except Exception as e: + logger.warning(f"Failed to broadcast status update for job reset {job_id}: {e}") + # Trigger ingestion task from ...tasks.ingest_and_ai import ingest_and_ai_task ingest_and_ai_task.delay(job_id) diff --git a/backend/app/api/v1/routes_jobs.py b/backend/app/api/v1/routes_jobs.py index 26d943e..63ab65c 100644 --- a/backend/app/api/v1/routes_jobs.py +++ b/backend/app/api/v1/routes_jobs.py @@ -352,17 +352,6 @@ async def approve_english( detail="Job not found or not in pending QC status" ) - # Broadcast status update - try: - await connection_manager.broadcast_job_status_update( - job_id=job_id, - status=JobStatus.APPROVED_ENGLISH.value, - message="English content approved - starting translation", - progress=None - ) - except Exception as e: - logger.warning(f"Failed to broadcast status update for job {job_id}: {e}") - # Trigger translation and synthesis pipeline immediately try: translate_and_synthesize_task.delay(job_id) @@ -418,17 +407,6 @@ async def reject_job( detail="Job not found or not in pending QC status" ) - # Broadcast status update - try: - await connection_manager.broadcast_job_status_update( - job_id=job_id, - status=JobStatus.REJECTED.value, - message="Job rejected - requires revision", - progress=None - ) - except Exception as e: - logger.warning(f"Failed to broadcast status update for job {job_id}: {e}") - return JobResponse( id=str(result["_id"]), title=result["title"], @@ -497,16 +475,14 @@ async def complete_job( detail="Job not found or not in pending final review status" ) - # Broadcast status update + + # Trigger client notification task now that job is completed try: - await connection_manager.broadcast_job_status_update( - job_id=job_id, - status=JobStatus.COMPLETED.value, - message="Job completed - all files ready for download", - progress=100 - ) + from ...tasks.notify import notify_client_task + notify_client_task.delay(job_id) + logger.info(f"Triggered client notification task for completed job {job_id}") except Exception as e: - logger.warning(f"Failed to broadcast status update for job {job_id}: {e}") + logger.warning(f"Failed to trigger client notification task for job {job_id}: {e}") return JobResponse( id=str(result["_id"]), @@ -555,16 +531,6 @@ async def reject_final_review( detail="Job not found or not in pending final review status" ) - # Broadcast status update - try: - await connection_manager.broadcast_job_status_update( - job_id=job_id, - status=JobStatus.QC_FEEDBACK.value, - message="Final review rejected - requires changes", - progress=None - ) - except Exception as e: - logger.warning(f"Failed to broadcast status update for job {job_id}: {e}") return JobResponse( id=str(result["_id"]), diff --git a/backend/app/services/__pycache__/websocket.cpython-313.pyc b/backend/app/services/__pycache__/websocket.cpython-313.pyc index 84fe2a5..7b256d8 100644 Binary files a/backend/app/services/__pycache__/websocket.cpython-313.pyc and b/backend/app/services/__pycache__/websocket.cpython-313.pyc differ diff --git a/backend/app/services/websocket.py b/backend/app/services/websocket.py index e2aabc7..4e055c3 100644 --- a/backend/app/services/websocket.py +++ b/backend/app/services/websocket.py @@ -28,6 +28,7 @@ class JobStatusUpdate(BaseModel): job_id: str status: str updated_at: datetime + job_title: Optional[str] = None # Job title for better user experience message: Optional[str] = None progress: Optional[int] = None # 0-100 percentage metadata: Optional[Dict[str, Any]] = None @@ -157,6 +158,7 @@ class ConnectionManager: self, job_id: str, status: str, + job_title: Optional[str] = None, user_id: Optional[str] = None, message: Optional[str] = None, progress: Optional[int] = None, @@ -170,6 +172,7 @@ class ConnectionManager: job_id=job_id, status=status, updated_at=datetime.utcnow(), + job_title=job_title, message=message, progress=progress, metadata=metadata @@ -255,15 +258,68 @@ class ConnectionManager: await self._send_to_user(user_id, message) async def _send_job_status_to_global_subscribers(self, update: JobStatusUpdate): - """Send job status update to global (job list) subscribers""" + """Send job status update to global (job list) subscribers with user filtering""" # Convert to JSON-serializable dict message = { "type": "job_list_update", "data": json.loads(update.model_dump_json()) } + # Get users who should receive this notification + eligible_users = await self._get_job_related_users(update.job_id) + + # Only send to users who are both subscribed and have access to this job for user_id in list(self.global_subscriptions): - await self._send_to_user(user_id, message) + if user_id in eligible_users: + await self._send_to_user(user_id, message) + + async def _get_job_related_users(self, job_id: str) -> Set[str]: + """ + Get all users who should receive notifications for a specific job. + Returns set of user IDs for: + - Job creator (client_id) + - Reviewers who worked on the job + - Admin users (see all jobs) + """ + eligible_users = set() + + try: + # Import database connection + from ..core.database import get_database + db = await get_database() + + # Get the job + job = await db["jobs"].find_one({"_id": job_id}) + if not job: + logger.warning(f"Job {job_id} not found for notification filtering") + return eligible_users + + # Add job creator + if job.get("client_id"): + eligible_users.add(job["client_id"]) + + # Add reviewers from review history + review = job.get("review", {}) + if review.get("reviewer_id"): + eligible_users.add(review["reviewer_id"]) + + # Add reviewers from history + for history_item in review.get("history", []): + if history_item.get("by"): + eligible_users.add(history_item["by"]) + + # Add all admin users (they can see all jobs) + admin_users = db["users"].find({"role": "admin"}) + async for admin_user in admin_users: + user_id = str(admin_user["_id"]) + eligible_users.add(user_id) + + logger.debug(f"Job {job_id} notification eligible users: {len(eligible_users)}") + + except Exception as e: + logger.error(f"Error getting job related users for {job_id}: {e}") + + return eligible_users async def _send_to_user(self, user_id: str, message: Dict[str, Any]): """Send message to all WebSocket connections for a user""" diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py index cfee0d7..032afd6 100644 --- a/backend/app/tasks/__init__.py +++ b/backend/app/tasks/__init__.py @@ -28,21 +28,11 @@ celery_app.conf.update( "app.tasks.ingest_and_ai.*": {"queue": "ingest"}, "app.tasks.translate_and_synthesize.*": {"queue": "default"}, "app.tasks.notify.*": {"queue": "notify"}, - "app.tasks.watchers.*": {"queue": "default"}, }, task_default_queue="default", task_create_missing_queues=True, - # Task-specific timeout overrides - task_annotations={ - 'app.tasks.watchers.start_change_stream_watcher': { - 'time_limit': None, - 'soft_time_limit': None, - }, - 'app.tasks.watchers.ensure_watcher_running': { - 'time_limit': 300, # 5 minutes - 'soft_time_limit': 240, # 4 minutes - }, - }, + # Task-specific timeout overrides + task_annotations={}, ) # Add a simple test task for debugging @@ -65,18 +55,8 @@ def worker_ready_handler(sender=None, **kwargs): logger.info(f"🟢 WORKER READY: {sender}") print(f"🟢 WORKER READY: {sender} - Worker is online and listening!") - # Start MongoDB change stream watcher - # Note: The main job progression is handled by immediate triggering in approve_english endpoint - # This watcher provides redundancy for status change detection - if _watchers_available and 'app.tasks.watchers.ensure_watcher_running' in celery_app.tasks: - try: - from .watchers import ensure_watcher_running - ensure_watcher_running.apply_async(countdown=3) # Start after 3 seconds - logger.info("Scheduled MongoDB change stream watcher to start") - except Exception as e: - logger.error(f"Failed to schedule change stream watcher: {e}") - else: - logger.info("Watcher not available or not registered, using primary job progression via approve_english endpoint") + # Change stream monitoring has been removed - workflow triggering now handled directly by API endpoints + logger.info("Workflow triggering handled directly by API endpoints - no change stream monitoring needed") @task_received.connect @@ -138,21 +118,9 @@ def import_task_modules(): from . import ingest_and_ai # noqa: E402, F401 from . import translate_and_synthesize # noqa: E402, F401 from . import notify # noqa: E402, F401 - logger.info("Successfully imported core task modules") + logger.info("Successfully imported all task modules") except Exception as e: - logger.error(f"Error importing core task modules: {e}") - - # Import watchers module conditionally to handle import errors gracefully - try: - from . import watchers # noqa: E402, F401 - logger.info("Successfully imported watchers module") - return True - except ImportError as e: - logger.warning(f"Could not import watchers module: {e}") - return False - except Exception as e: - logger.error(f"Error importing watchers module: {e}") - return False + logger.error(f"Error importing task modules: {e}") # Import task modules at startup -_watchers_available = import_task_modules() +import_task_modules() diff --git a/backend/app/tasks/__pycache__/__init__.cpython-313.pyc b/backend/app/tasks/__pycache__/__init__.cpython-313.pyc index 04eadb9..29fc428 100644 Binary files a/backend/app/tasks/__pycache__/__init__.cpython-313.pyc and b/backend/app/tasks/__pycache__/__init__.cpython-313.pyc differ diff --git a/backend/app/tasks/__pycache__/ingest_and_ai.cpython-313.pyc b/backend/app/tasks/__pycache__/ingest_and_ai.cpython-313.pyc index 09d57ef..04d8c72 100644 Binary files a/backend/app/tasks/__pycache__/ingest_and_ai.cpython-313.pyc and b/backend/app/tasks/__pycache__/ingest_and_ai.cpython-313.pyc differ diff --git a/backend/app/tasks/__pycache__/translate_and_synthesize.cpython-313.pyc b/backend/app/tasks/__pycache__/translate_and_synthesize.cpython-313.pyc index 7877ba6..b1f447e 100644 Binary files a/backend/app/tasks/__pycache__/translate_and_synthesize.cpython-313.pyc and b/backend/app/tasks/__pycache__/translate_and_synthesize.cpython-313.pyc differ diff --git a/backend/app/tasks/__pycache__/watchers.cpython-313.pyc b/backend/app/tasks/__pycache__/watchers.cpython-313.pyc index f2d74f8..f32320b 100644 Binary files a/backend/app/tasks/__pycache__/watchers.cpython-313.pyc and b/backend/app/tasks/__pycache__/watchers.cpython-313.pyc differ diff --git a/backend/app/tasks/ingest_and_ai.py b/backend/app/tasks/ingest_and_ai.py index d5e3eb3..4f6ed71 100644 --- a/backend/app/tasks/ingest_and_ai.py +++ b/backend/app/tasks/ingest_and_ai.py @@ -18,12 +18,12 @@ from . import celery_app logger = get_logger(__name__) -def broadcast_status_update(job_id: str, status: str, message: str = None, progress: int = None): +def broadcast_status_update(job_id: str, status: str, job_title: str = None, message: str = None, progress: int = None): """ Helper function to broadcast job status updates via WebSocket Uses sync Redis client for Celery worker context """ - logger.info(f"🔊 ATTEMPTING TO BROADCAST: job_id={job_id}, status={status}, message={message}") + logger.info(f"🔊 ATTEMPTING TO BROADCAST: job_id={job_id}, status={status}, job_title={job_title}, message={message}") try: import redis as sync_redis from ..core.config import settings @@ -37,6 +37,7 @@ def broadcast_status_update(job_id: str, status: str, message: str = None, progr job_id=job_id, status=status, updated_at=datetime.utcnow(), + job_title=job_title, message=message, progress=progress ) @@ -115,6 +116,15 @@ async def ingest_and_ai_task_impl(job_id: str): db = client[settings.mongodb_db] try: + # Get job document to retrieve title for notifications + job_doc = await db.jobs.find_one({"_id": job_id}) + if not job_doc: + logger.error(f"Job {job_id} not found in database") + return + + job_title = job_doc.get("title", "Untitled Job") + logger.info(f"Processing job: {job_title}") + # Update status to ingesting await db.jobs.update_one( {"_id": job_id}, @@ -132,14 +142,6 @@ async def ingest_and_ai_task_impl(job_id: str): } } ) - - # Broadcast status update - broadcast_status_update( - job_id, - JobStatus.INGESTING.value, - "Starting video ingestion and processing", - progress=10 - ) # Get job details job_doc = await db.jobs.find_one({"_id": job_id}) @@ -174,14 +176,6 @@ async def ingest_and_ai_task_impl(job_id: str): } } ) - - # Broadcast status update - broadcast_status_update( - job_id, - JobStatus.AI_PROCESSING.value, - "Processing video with AI for accessibility features", - progress=50 - ) # Probe video for metadata duration = await _get_video_duration(temp_path) @@ -244,9 +238,9 @@ async def ingest_and_ai_task_impl(job_id: str): # Broadcast status update broadcast_status_update( job_id, - JobStatus.PENDING_QC.value, - "AI processing complete - ready for quality review", - progress=100 + JobStatus.PENDING_QC.value, + job_title=job_title, + message=f"{job_title} has completed AI processing and is ready for QC review" ) logger.info(f"Successfully completed ingestion and AI processing for job {job_id}") diff --git a/backend/app/tasks/translate_and_synthesize.py b/backend/app/tasks/translate_and_synthesize.py index d44f5eb..aa4248a 100644 --- a/backend/app/tasks/translate_and_synthesize.py +++ b/backend/app/tasks/translate_and_synthesize.py @@ -20,12 +20,12 @@ from . import celery_app logger = get_logger(__name__) -def broadcast_status_update(job_id: str, status: str, message: str = None, progress: int = None): +def broadcast_status_update(job_id: str, status: str, job_title: str = None, message: str = None, progress: int = None): """ Helper function to broadcast job status updates via WebSocket Uses sync Redis client for Celery worker context """ - logger.info(f"🔊 ATTEMPTING TO BROADCAST: job_id={job_id}, status={status}, message={message}") + logger.info(f"🔊 ATTEMPTING TO BROADCAST: job_id={job_id}, status={status}, job_title={job_title}, message={message}") try: import redis as sync_redis from ..core.config import settings @@ -39,6 +39,7 @@ def broadcast_status_update(job_id: str, status: str, message: str = None, progr job_id=job_id, status=status, updated_at=datetime.utcnow(), + job_title=job_title, message=message, progress=progress ) @@ -131,7 +132,8 @@ async def _async_translate_and_synthesize(job_id: str): logger.error(f"❌ Job {job_id} not found in database!") raise ValueError(f"Job {job_id} not found") - logger.info(f"✅ Found job document for {job_id}, status: {job_doc.get('status', 'UNKNOWN')}") + job_title = job_doc.get("title", "Untitled Job") + logger.info(f"✅ Found job document for {job_id} ({job_title}), status: {job_doc.get('status', 'UNKNOWN')}") if job_doc["status"] != JobStatus.APPROVED_ENGLISH.value: logger.warning(f"⚠️ Job {job_id} not in approved_english status (current: {job_doc['status']}), skipping") @@ -156,14 +158,6 @@ async def _async_translate_and_synthesize(job_id: str): } } ) - - # Broadcast status update - broadcast_status_update( - job_id, - JobStatus.TRANSLATING.value, - "Starting translation and transcreation process", - progress=10 - ) # Get English VTT content en_outputs = job_doc["outputs"]["en"] @@ -263,14 +257,6 @@ async def _async_translate_and_synthesize(job_id: str): } } ) - - # Broadcast status update - broadcast_status_update( - job_id, - JobStatus.TTS_GENERATING.value, - "Generating audio descriptions with text-to-speech", - progress=70 - ) # Generate TTS for languages that need MP3 if job_doc["requested_outputs"]["audio_description_mp3"]: @@ -298,8 +284,8 @@ async def _async_translate_and_synthesize(job_id: str): broadcast_status_update( job_id, JobStatus.PENDING_FINAL_REVIEW.value, - "Translation and TTS complete - ready for final review", - progress=100 + job_title=job_title, + message=f"{job_title} has finished translation and audio generation - ready for Final Review" ) logger.info(f"Successfully completed translation and synthesis for job {job_id}") diff --git a/backend/app/tasks/watchers.py b/backend/app/tasks/watchers.py deleted file mode 100644 index 14ac323..0000000 --- a/backend/app/tasks/watchers.py +++ /dev/null @@ -1,136 +0,0 @@ -import asyncio - -from motor.motor_asyncio import AsyncIOMotorClient - -from ..core.config import settings -from ..core.logging import get_logger -from ..models.job import JobStatus -from . import celery_app - -logger = get_logger(__name__) - - -@celery_app.task( - bind=True, - acks_late=True, # Acknowledge task only after completion - reject_on_worker_lost=True, # Retry if worker crashes - autoretry_for=(Exception,), # Auto-retry on any exception - retry_kwargs={'max_retries': None, 'countdown': 60}, # Infinite retries with 60s delay - retry_backoff=True, # Exponential backoff -) -def start_change_stream_watcher(self): - """Start MongoDB change stream watcher for job status changes""" - try: - asyncio.run(_watch_job_changes()) - except Exception as e: - logger.error(f"Change stream watcher failed: {e}") - # Task will auto-retry due to configuration - raise - - -async def _watch_job_changes(): - """Watch MongoDB change streams for job status updates""" - client = AsyncIOMotorClient(settings.mongodb_uri) - db = client[settings.mongodb_db] - - logger.info("Starting MongoDB change stream watcher") - - try: - # Add a heartbeat mechanism to ensure the connection stays alive - await client.admin.command('ping') - logger.info("MongoDB connection verified") - # Watch for changes to the jobs collection - pipeline = [ - { - "$match": { - "operationType": "update", - "fullDocument.status": { - "$in": [ - JobStatus.APPROVED_ENGLISH.value, - JobStatus.COMPLETED.value - ] - } - } - } - ] - - async with db.jobs.watch( - pipeline, - full_document="updateLookup", - max_await_time_ms=30000, # 30 second timeout for getMore operations - batch_size=10 # Process changes in small batches - ) as stream: - logger.info("Change stream watcher active, waiting for job status changes...") - - async for change in stream: - try: - job_doc = change["fullDocument"] - if not job_doc: - logger.warning("Received change event without fullDocument") - continue - - job_id = str(job_doc["_id"]) - status = job_doc["status"] - - logger.info(f"Job {job_id} status changed to {status}") - - if status == JobStatus.APPROVED_ENGLISH.value: - # Trigger translation and synthesis - from .translate_and_synthesize import translate_and_synthesize_task - translate_and_synthesize_task.delay(job_id) - logger.info(f"Enqueued translation task for job {job_id}") - - elif status == JobStatus.COMPLETED.value: - # Trigger client notification - from .notify import notify_client_task - notify_client_task.delay(job_id) - logger.info(f"Enqueued notification task for job {job_id}") - - except Exception as e: - logger.error(f"Error processing change stream event: {e}") - # Continue processing other events - continue - - except Exception as e: - error_msg = str(e) - if "replica sets" in error_msg: - logger.warning("Change stream watcher not available - MongoDB not configured as replica set") - logger.info("This is normal in development. Job progression works via immediate triggering in approval endpoint.") - else: - logger.error(f"Change stream watcher failed: {e}") - # Don't re-raise in development to prevent worker crashes - - finally: - client.close() - - -# Auto-start the watcher when the worker starts -@celery_app.task( - bind=True, - autoretry_for=(Exception,), - retry_kwargs={'max_retries': 3, 'countdown': 30} -) -def ensure_watcher_running(self): - """Ensure the change stream watcher is running""" - try: - # Check if watcher is already running - active_tasks = celery_app.control.inspect().active() - - if not active_tasks: - logger.warning("Could not inspect active tasks - starting watcher anyway") - else: - # Look for running watcher - for worker, tasks in active_tasks.items(): - if tasks: # Check if tasks list is not None - for task in tasks: - if task.get("name") == "app.tasks.watchers.start_change_stream_watcher": - logger.info(f"Change stream watcher already running on worker {worker}") - return - - # Start the watcher - result = start_change_stream_watcher.delay() - logger.info(f"Started change stream watcher with task ID: {result.id}") - - except Exception as e: - logger.error(f"Failed to ensure watcher is running: {e}") - raise # Will trigger retry diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..4837684 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,110 @@ +version: '3.8' + +services: + # MongoDB with Replica Set + mongodb: + image: mongo:7.0 + container_name: accessible-video-mongo-prod + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGODB_ROOT_USER:-admin} + MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_ROOT_PASSWORD} + MONGO_INITDB_DATABASE: accessible_video + ports: + - "27017:27017" + volumes: + - mongodb_data_prod:/data/db + - ./mongo-init.js:/docker-entrypoint-initdb.d/init.js:ro + - ./mongo-keyfile:/data/keyfile:ro + command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--keyFile", "/data/keyfile"] + networks: + - app-network-prod + + # Redis + redis: + image: redis:7.2-alpine + container_name: accessible-video-redis-prod + restart: unless-stopped + ports: + - "6379:6379" + volumes: + - redis_data_prod:/data + networks: + - app-network-prod + + # Backend API + api: + build: + context: ./backend + dockerfile: Dockerfile + target: production + container_name: accessible-video-api-prod + restart: unless-stopped + environment: + - APP_ENV=production + - MONGODB_URL=${MONGODB_URL} + - REDIS_URL=${REDIS_URL} + - JWT_SECRET_KEY=${JWT_SECRET_KEY} + - JWT_REFRESH_SECRET_KEY=${JWT_REFRESH_SECRET_KEY} + - GEMINI_API_KEY=${GEMINI_API_KEY} + - SENDGRID_API_KEY=${SENDGRID_API_KEY} + - ELEVENLABS_API_KEY=${ELEVENLABS_API_KEY} + - GCS_BUCKET_NAME=${GCS_BUCKET_NAME} + - GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} + - OTEL_SERVICE_NAME=accessible-video-api-prod + - SENTRY_DSN=${SENTRY_DSN} + - SENTRY_ENVIRONMENT=production + - CORS_ORIGINS=${CORS_ORIGINS:-https://your-domain.com,https://www.your-domain.com} + ports: + - "8000:8000" + depends_on: + - mongodb + - redis + networks: + - app-network-prod + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + + # Celery Worker + worker: + build: + context: ./backend + dockerfile: Dockerfile + target: production + container_name: accessible-video-worker-prod + restart: unless-stopped + environment: + - APP_ENV=production + - MONGODB_URL=${MONGODB_URL} + - REDIS_URL=${REDIS_URL} + - CELERY_BROKER_URL=${REDIS_URL} + - CELERY_RESULT_BACKEND=${REDIS_URL} + - GEMINI_API_KEY=${GEMINI_API_KEY} + - ELEVENLABS_API_KEY=${ELEVENLABS_API_KEY} + - GCS_BUCKET_NAME=${GCS_BUCKET_NAME} + - GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} + - OTEL_SERVICE_NAME=accessible-video-worker-prod + - SENTRY_DSN=${SENTRY_DSN} + - SENTRY_ENVIRONMENT=production + depends_on: + - mongodb + - redis + command: ["celery", "-A", "app.tasks", "worker", "--loglevel=info", "--concurrency=2"] + networks: + - app-network-prod + + + # Note: Frontend will be built separately and hosted on Apache webserver + # Build command: cd frontend && npm run build + # Deploy the 'dist' folder contents to your Apache document root + +volumes: + mongodb_data_prod: + redis_data_prod: + +networks: + app-network-prod: + driver: bridge \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bafe345..4513526 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,7 @@ services: networks: - app-network + # Frontend (for local development) frontend: build: diff --git a/docs/video_accessibility_app_demo_2025-08-24.mp4 b/docs/video_accessibility_app_demo_2025-08-24.mp4 new file mode 100644 index 0000000..8daada1 Binary files /dev/null and b/docs/video_accessibility_app_demo_2025-08-24.mp4 differ diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7a2cbc9..83a3947 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -21,6 +21,7 @@ import { ErrorBoundary } from './components/ErrorBoundary'; import { ToastContainer } from './components/Toast/Toast'; import { ToastProvider, useToastContext } from './contexts/ToastContext'; import { NotificationProvider } from './contexts/NotificationContext'; +import { GlobalWebSocketProvider } from './contexts/GlobalWebSocketContext'; import { Layout } from './components/Layout/Layout'; // Helper component to wrap authenticated routes with Layout @@ -108,7 +109,9 @@ function App() { - + + + diff --git a/frontend/src/components/WebSocketToastHandler.tsx b/frontend/src/components/WebSocketToastHandler.tsx index 35ececc..c9a6116 100644 --- a/frontend/src/components/WebSocketToastHandler.tsx +++ b/frontend/src/components/WebSocketToastHandler.tsx @@ -35,7 +35,7 @@ export function useWebSocketToastHandler(props: WebSocketToastHandlerProps = {}) switch (status) { case 'connected': - toast.success(messages.connected || 'Real-time updates connected'); + toast.toastOnly.success(messages.connected || 'Real-time updates connected'); break; case 'connecting': @@ -44,11 +44,11 @@ export function useWebSocketToastHandler(props: WebSocketToastHandlerProps = {}) break; case 'disconnected': - toast.warning(messages.disconnected || 'Real-time updates disconnected'); + toast.toastOnly.warning(messages.disconnected || 'Real-time updates disconnected'); break; case 'error': - toast.error(messages.error || 'Connection error - using cached data'); + toast.toastOnly.error(messages.error || 'Connection error - using cached data'); break; } }, [enabled, messages, toast]); diff --git a/frontend/src/contexts/GlobalWebSocketContext.tsx b/frontend/src/contexts/GlobalWebSocketContext.tsx new file mode 100644 index 0000000..dab32f9 --- /dev/null +++ b/frontend/src/contexts/GlobalWebSocketContext.tsx @@ -0,0 +1,61 @@ +import { createContext, useContext, ReactNode, useCallback } from 'react'; +import { useJobStatusWebSocket } from '../hooks/useJobStatusWebSocket'; +import { useToastContext } from './ToastContext'; +import { getStatusMessageConfig } from '../utils/jobStatusMessages'; +import type { JobStatusUpdate, ConnectionStatus } from '../hooks/useJobStatusWebSocket'; + +interface GlobalWebSocketContextType { + connectionStatus: ConnectionStatus; + reconnect: () => void; + disconnect: () => void; +} + +const GlobalWebSocketContext = createContext(undefined); + +export function GlobalWebSocketProvider({ children }: { children: ReactNode }) { + const toast = useToastContext(); + + // Handle job status updates globally + const handleStatusUpdate = useCallback((update: JobStatusUpdate) => { + // Use job_title from the update, or fallback to generic message + const jobTitle = update.job_title || 'Job'; + + const { message, type, showToast } = getStatusMessageConfig( + update.status, + jobTitle, + update.message + ); + + if (showToast) { + // Pass job_id and job_title to toast for notification integration + toast[type](message, undefined, update.job_id, update.job_title); + } + }, [toast]); + + // Use the existing WebSocket hook for global job list updates (no jobId) + const { connectionStatus, reconnect, disconnect } = useJobStatusWebSocket(undefined, { + debug: false, + autoReconnect: true, + onStatusUpdate: handleStatusUpdate + }); + + const contextValue: GlobalWebSocketContextType = { + connectionStatus, + reconnect, + disconnect + }; + + return ( + + {children} + + ); +} + +export const useGlobalWebSocket = () => { + const context = useContext(GlobalWebSocketContext); + if (context === undefined) { + throw new Error('useGlobalWebSocket must be used within a GlobalWebSocketProvider'); + } + return context; +}; \ No newline at end of file diff --git a/frontend/src/contexts/ToastContext.tsx b/frontend/src/contexts/ToastContext.tsx index 45fe36b..2a08c31 100644 --- a/frontend/src/contexts/ToastContext.tsx +++ b/frontend/src/contexts/ToastContext.tsx @@ -12,6 +12,13 @@ interface ToastContextType { warning: (message: string, duration?: number, jobId?: string, jobTitle?: string) => string; info: (message: string, duration?: number, jobId?: string, jobTitle?: string) => string; clearAll: () => void; + // Toast-only methods (no persistent notifications) + toastOnly: { + success: (message: string, duration?: number) => string; + error: (message: string, duration?: number) => string; + warning: (message: string, duration?: number) => string; + info: (message: string, duration?: number) => string; + }; } const ToastContext = createContext(undefined); @@ -48,6 +55,21 @@ export function ToastProvider({ children }: { children: ReactNode }) { notifications.addNotification(message, 'info', jobId, jobTitle); return toastId; }, + // Toast-only methods (no persistent notifications) + toastOnly: { + success: (message: string, duration?: number) => { + return toastMethods.success(message, duration); + }, + error: (message: string, duration?: number) => { + return toastMethods.error(message, duration); + }, + warning: (message: string, duration?: number) => { + return toastMethods.warning(message, duration); + }, + info: (message: string, duration?: number) => { + return toastMethods.info(message, duration); + }, + }, }; return ( diff --git a/frontend/src/hooks/useJob.ts b/frontend/src/hooks/useJob.ts index de16c61..796a657 100644 --- a/frontend/src/hooks/useJob.ts +++ b/frontend/src/hooks/useJob.ts @@ -193,4 +193,17 @@ export function useAdjustVttTiming() { queryClient.invalidateQueries({ queryKey: ['jobs', id, 'vtt', language] }); }, }); +} + +export function useReprocessJob() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: (id: string) => apiClient.reprocessJob(id), + onSuccess: (_, id) => { + // Invalidate both the specific job and the jobs list to reflect status changes + queryClient.invalidateQueries({ queryKey: ['jobs', id] }); + queryClient.invalidateQueries({ queryKey: ['jobs'] }); + }, + }); } \ No newline at end of file diff --git a/frontend/src/hooks/useJobStatusWebSocket.ts b/frontend/src/hooks/useJobStatusWebSocket.ts index c86640d..bebfc0d 100644 --- a/frontend/src/hooks/useJobStatusWebSocket.ts +++ b/frontend/src/hooks/useJobStatusWebSocket.ts @@ -14,6 +14,7 @@ export interface JobStatusUpdate { job_id: string; status: string; updated_at: string; + job_title?: string; message?: string; progress?: number; metadata?: Record; diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 71be4b3..f5e5660 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -211,6 +211,11 @@ class ApiClient { const response = await this.client.delete('/jobs/bulk', { data }); return response.data; } + + async reprocessJob(id: string): Promise<{ message: string }> { + const response = await this.client.post(`/admin/maintenance/reprocess-job/${id}`); + return response.data; + } } export const apiClient = new ApiClient(); diff --git a/frontend/src/routes/admin/QCDetail.tsx b/frontend/src/routes/admin/QCDetail.tsx index fb86e1b..1aead28 100644 --- a/frontend/src/routes/admin/QCDetail.tsx +++ b/frontend/src/routes/admin/QCDetail.tsx @@ -108,10 +108,10 @@ export function QCDetail() { } }); setHasUnsavedChanges(false); - toast.success('VTT content saved successfully'); + toast.toastOnly.success('VTT content saved successfully'); } catch (error) { console.error('Failed to save VTT changes:', error); - toast.error('Failed to save VTT content. Please try again.'); + toast.toastOnly.error('Failed to save VTT content. Please try again.'); } }; @@ -138,11 +138,11 @@ export function QCDetail() { id, notes: reviewNotes }); - toast.success('Job approved successfully'); + toast.toastOnly.success('Job approved successfully'); navigate('/admin/qc'); } catch (error) { console.error('Approval failed:', error); - toast.error('Failed to approve job. Please try again.'); + toast.toastOnly.error('Failed to approve job. Please try again.'); } }; @@ -154,11 +154,11 @@ export function QCDetail() { id, notes: reviewNotes }); - toast.success('Job rejected and returned for revision'); + toast.toastOnly.success('Job rejected and returned for revision'); navigate('/admin/qc'); } catch (error) { console.error('Rejection failed:', error); - toast.error('Failed to reject job. Please try again.'); + toast.toastOnly.error('Failed to reject job. Please try again.'); } }; @@ -173,12 +173,12 @@ export function QCDetail() { adjustCaptions, adjustAudioDescription, }); - toast.success(`Timing adjusted by ${timingOffset > 0 ? '+' : ''}${timingOffset} seconds`); + toast.toastOnly.success(`Timing adjusted by ${timingOffset > 0 ? '+' : ''}${timingOffset} seconds`); setShowTimingAdjustment(false); setTimingOffset(0); } catch (error) { console.error('Timing adjustment failed:', error); - toast.error('Failed to adjust timing. Please try again.'); + toast.toastOnly.error('Failed to adjust timing. Please try again.'); } }; diff --git a/frontend/src/routes/admin/QCList.tsx b/frontend/src/routes/admin/QCList.tsx index 7adb2b6..d59958a 100644 --- a/frontend/src/routes/admin/QCList.tsx +++ b/frontend/src/routes/admin/QCList.tsx @@ -65,7 +65,7 @@ export function QCList() { const handleBulkAction = async () => { if (!bulkAction || selectedJobs.size === 0) return; if (bulkAction === 'reject' && !bulkNotes.trim()) { - toast.error('Please provide notes for rejection'); + toast.toastOnly.error('Please provide notes for rejection'); return; } @@ -82,13 +82,13 @@ export function QCList() { try { await Promise.all(promises); - toast.success(`${jobCount} job${jobCount > 1 ? 's' : ''} ${actionText} successfully`); + toast.toastOnly.success(`${jobCount} job${jobCount > 1 ? 's' : ''} ${actionText} successfully`); clearSelection(); setBulkAction(''); setBulkNotes(''); } catch (error) { console.error('Bulk action failed:', error); - toast.error(`Failed to ${bulkAction} selected jobs. Some may have been processed.`); + toast.toastOnly.error(`Failed to ${bulkAction} selected jobs. Some may have been processed.`); } }; diff --git a/frontend/src/routes/jobs/JobDetail.tsx b/frontend/src/routes/jobs/JobDetail.tsx index bab8141..a87e465 100644 --- a/frontend/src/routes/jobs/JobDetail.tsx +++ b/frontend/src/routes/jobs/JobDetail.tsx @@ -1,13 +1,10 @@ -import { useState, useCallback } from 'react'; +import { useState } from 'react'; import { useParams, Link } from 'react-router-dom'; import { formatDistanceToNow } from 'date-fns'; import { useJob, useJobDownloads, useJobVttContent } from '../../hooks/useJob'; -import { useJobStatusWebSocket } from '../../hooks/useJobStatusWebSocket'; import { StatusBadge } from '../../components/StatusBadge'; import { VideoWithCaptions } from '../../components/VideoWithCaptions'; -import { useToastContext } from '../../contexts/ToastContext'; -import { getStatusMessageConfig } from '../../utils/jobStatusMessages'; -import type { JobStatusUpdate } from '../../hooks/useJobStatusWebSocket'; +import { useGlobalWebSocket } from '../../contexts/GlobalWebSocketContext'; const ProgressIndicator = ({ status }: { status: string }) => { @@ -62,34 +59,13 @@ const ProgressIndicator = ({ status }: { status: string }) => { export function JobDetail() { const { id } = useParams(); const [activeTab, setActiveTab] = useState<'overview' | 'video' | 'assets' | 'history'>('overview'); - const toast = useToastContext(); const { data: job, isLoading, error } = useJob(id!); const { data: downloads } = useJobDownloads(id!); const { data: englishVtt } = useJobVttContent(id!, 'en'); - // Handle job status updates with toast notifications - const handleStatusUpdate = useCallback((update: JobStatusUpdate) => { - // Use the current job title or fallback - const jobTitle = job?.title; - - const { message, type, showToast } = getStatusMessageConfig( - update.status, - jobTitle, - update.message - ); - - if (showToast) { - toast[type](message, undefined, update.job_id, jobTitle); - } - }, [job?.title, toast]); - - // WebSocket connection for real-time job status updates - const { connectionStatus } = useJobStatusWebSocket(id, { - debug: false, - autoReconnect: true, - onStatusUpdate: handleStatusUpdate - }); + // Get connection status from global WebSocket + const { connectionStatus } = useGlobalWebSocket(); // Get video URL from downloads diff --git a/frontend/src/routes/jobs/JobsList.tsx b/frontend/src/routes/jobs/JobsList.tsx index cb65ff2..eacd1b6 100644 --- a/frontend/src/routes/jobs/JobsList.tsx +++ b/frontend/src/routes/jobs/JobsList.tsx @@ -1,14 +1,12 @@ -import { useState, useMemo, useEffect, useCallback } from 'react'; +import { useState, useMemo, useEffect } from 'react'; import { Link, useSearchParams } from 'react-router-dom'; import { formatDistanceToNow } from 'date-fns'; import { useAuthStore } from '../../lib/auth'; -import { useJobs, useBulkDeleteJobs } from '../../hooks/useJob'; -import { useJobStatusWebSocket } from '../../hooks/useJobStatusWebSocket'; +import { useJobs, useBulkDeleteJobs, useReprocessJob } from '../../hooks/useJob'; import { StatusBadge } from '../../components/StatusBadge'; import { useToastContext } from '../../contexts/ToastContext'; -import { getStatusMessageConfig } from '../../utils/jobStatusMessages'; +import { useGlobalWebSocket } from '../../contexts/GlobalWebSocketContext'; import type { Job } from '../../types/api'; -import type { JobStatusUpdate } from '../../hooks/useJobStatusWebSocket'; const STATUS_OPTIONS = [ { value: '', label: 'All Statuses' }, @@ -37,8 +35,9 @@ export function JobsList() { const [sortBy, setSortBy] = useState('created_at_desc'); const [currentPage, setCurrentPage] = useState(1); const [selectedJobs, setSelectedJobs] = useState>(new Set()); - const [bulkAction, setBulkAction] = useState<'delete' | ''>(''); + const [bulkAction, setBulkAction] = useState<'delete' | 'reprocess' | ''>(''); const [showDeleteConfirm, setShowDeleteConfirm] = useState(false); + const [showReprocessConfirm, setShowReprocessConfirm] = useState(false); const pageSize = 20; // Update filters from URL params @@ -58,30 +57,10 @@ export function JobsList() { }); const bulkDeleteMutation = useBulkDeleteJobs(); + const reprocessMutation = useReprocessJob(); - // Handle job status updates with toast notifications - const handleStatusUpdate = useCallback((update: JobStatusUpdate) => { - // Find the job to get its title - const job = jobsData?.jobs.find((j: Job) => j.id === update.job_id); - const jobTitle = job?.title; - - const { message, type, showToast } = getStatusMessageConfig( - update.status, - jobTitle, - update.message - ); - - if (showToast) { - toast[type](message, undefined, update.job_id, jobTitle); - } - }, [jobsData?.jobs, toast]); - - // WebSocket connection for real-time job status updates - const { connectionStatus } = useJobStatusWebSocket(undefined, { - debug: false, - autoReconnect: true, - onStatusUpdate: handleStatusUpdate - }); + // Get connection status from global WebSocket + const { connectionStatus } = useGlobalWebSocket(); // Client-side filtering and sorting for search term const filteredAndSortedJobs = useMemo(() => { @@ -149,21 +128,61 @@ export function JobsList() { }); if (result.errors.length > 0) { - toast.warning(`${result.deleted_count}/${jobCount} jobs deleted. Some errors occurred: ${result.errors.join(', ')}`); + toast.toastOnly.warning(`${result.deleted_count}/${jobCount} jobs deleted. Some errors occurred: ${result.errors.join(', ')}`); } else { - toast.success(`${jobCount} job${jobCount > 1 ? 's' : ''} deleted successfully`); + toast.toastOnly.success(`${jobCount} job${jobCount > 1 ? 's' : ''} deleted successfully`); } clearSelection(); setShowDeleteConfirm(false); } catch (error) { console.error('Bulk delete failed:', error); - toast.error('Failed to delete jobs. Please try again.'); + toast.toastOnly.error('Failed to delete jobs. Please try again.'); + } + }; + + const handleReprocessConfirm = async () => { + if (selectedJobs.size === 0) return; + + const jobIds = Array.from(selectedJobs); + let successCount = 0; + let errorCount = 0; + const errors: string[] = []; + + try { + // Process jobs sequentially to avoid overwhelming the system + for (const jobId of jobIds) { + try { + await reprocessMutation.mutateAsync(jobId); + successCount++; + } catch (error) { + errorCount++; + errors.push(jobId); + console.error(`Failed to reprocess job ${jobId}:`, error); + } + } + + // Show appropriate feedback + if (successCount === jobIds.length) { + toast.success(`${successCount} job${successCount > 1 ? 's' : ''} queued for reprocessing`); + } else if (successCount > 0) { + toast.warning(`${successCount}/${jobIds.length} jobs queued for reprocessing. ${errorCount} failed.`); + } else { + toast.error('Failed to queue jobs for reprocessing. Please try again.'); + } + + clearSelection(); + setShowReprocessConfirm(false); + setBulkAction(''); + } catch (error) { + console.error('Bulk reprocess failed:', error); + toast.error('Failed to reprocess jobs. Please try again.'); } }; const isAdmin = user?.role === 'admin'; const canBulkDelete = isAdmin && selectedJobs.size > 0; + const canBulkReprocess = isAdmin && selectedJobs.size > 0; const getActionButton = (job: Job) => { const isReviewer = ['reviewer', 'admin'].includes(user?.role || ''); @@ -323,15 +342,16 @@ export function JobsList() { - {canBulkDelete && ( + {(canBulkDelete || canBulkReprocess) && ( <> {bulkAction === 'delete' && ( @@ -344,6 +364,16 @@ export function JobsList() { )} + {bulkAction === 'reprocess' && ( + + )} + + + + + + + )} ); } \ No newline at end of file diff --git a/frontend/src/routes/jobs/NewJob.tsx b/frontend/src/routes/jobs/NewJob.tsx index 2bd5a3c..83d12b9 100644 --- a/frontend/src/routes/jobs/NewJob.tsx +++ b/frontend/src/routes/jobs/NewJob.tsx @@ -51,7 +51,7 @@ export function NewJob() { const onSubmit = async (data: JobFormData) => { if (!selectedFile) { - toast.error('Please select a video file'); + toast.toastOnly.error('Please select a video file'); return; } @@ -89,7 +89,7 @@ export function NewJob() { }, 3000); } catch (error) { console.error('Job creation failed:', error); - toast.error('Failed to create job. Please check your file and try again.'); + toast.toastOnly.error('Failed to create job. Please check your file and try again.'); setUploadProgress(0); } }; diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..b77c521 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,135 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Logging + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log warn; + + # Performance + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_min_length 1000; + gzip_proxied any; + gzip_comp_level 6; + gzip_types + text/plain + text/css + text/xml + text/javascript + application/javascript + application/xml+rss + application/json; + + # Rate limiting + limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s; + limit_req_zone $binary_remote_addr zone=uploads:10m rate=2r/s; + + upstream backend { + server api:8000; + } + + server { + listen 80; + server_name localhost; + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # API routes + location /api/ { + limit_req zone=api burst=20 nodelay; + + proxy_pass http://backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # File upload size + client_max_body_size 500M; + } + + # WebSocket routes + location /api/v1/ws { + proxy_pass http://backend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # File uploads with special rate limiting + location /api/v1/files/upload { + limit_req zone=uploads burst=5 nodelay; + + proxy_pass http://backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Large file upload settings + client_max_body_size 1G; + proxy_connect_timeout 300s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + } + + # Frontend static files + location / { + root /usr/share/nginx/html; + try_files $uri $uri/ /index.html; + + # Cache static assets + location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { + expires 1y; + add_header Cache-Control "public, immutable"; + } + + # Don't cache HTML files + location ~* \.html$ { + expires -1; + add_header Cache-Control "no-cache, no-store, must-revalidate"; + } + } + + # Health check endpoint + location /health { + access_log off; + return 200 "OK\n"; + add_header Content-Type text/plain; + } + } +} \ No newline at end of file