feat: add RENDER_FAILED status with error propagation to GUI
- Add RENDER_FAILED job status for when video rendering fails - Fix _check_accessible_video_completion to detect failures and transition job status accordingly (was stuck in RENDERING_VIDEO forever) - Store detailed error info in job.error including failed_languages array - Call completion check after failures to properly update job status - Broadcast WebSocket notification on render failures Frontend: - Add render_failed to JobStatus type and StatusBadge (red styling) - Add tts_failed and render_failed to JobsList STATUS_LABELS - Enhance JobDetail error display with: - Warning icon and prominent styling - Error type and message - Failed languages list with per-language errors - Timestamp of when error occurred - Update ProgressIndicator to handle failed states with red dot 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
michael
parent
0c1c115a8f
commit
c1c0b876fc
6 changed files with 160 additions and 34 deletions
|
|
@ -18,6 +18,7 @@ class JobStatus(str, Enum):
|
|||
TTS_GENERATING = "tts_generating"
|
||||
TTS_FAILED = "tts_failed" # TTS synthesis failed after retries, requires reprocessing
|
||||
RENDERING_VIDEO = "rendering_video" # Accessible video rendering in progress
|
||||
RENDER_FAILED = "render_failed" # Accessible video rendering failed, requires reprocessing
|
||||
PENDING_FINAL_REVIEW = "pending_final_review"
|
||||
COMPLETED = "completed"
|
||||
|
||||
|
|
|
|||
|
|
@ -293,6 +293,10 @@ async def _async_render_accessible_video(job_id: str, language: str):
|
|||
}
|
||||
)
|
||||
|
||||
# Check if all videos are now finished (completed or failed) to update job status
|
||||
# This ensures the job transitions to RENDER_FAILED if all languages have finished
|
||||
await _check_accessible_video_completion(job_id, db)
|
||||
|
||||
raise
|
||||
|
||||
finally:
|
||||
|
|
@ -308,48 +312,95 @@ async def _check_accessible_video_completion(job_id: str, db):
|
|||
progress = job_doc.get("accessible_video_progress", {})
|
||||
requested_languages = job_doc["requested_outputs"]["languages"]
|
||||
|
||||
# Check if all requested languages have completed accessible video
|
||||
all_complete = True
|
||||
_any_failed = False
|
||||
# Check status of all requested languages
|
||||
all_finished = True # All languages have either completed or failed
|
||||
any_failed = False
|
||||
failed_languages = []
|
||||
|
||||
for language in requested_languages:
|
||||
lang_progress = progress.get(language, {})
|
||||
status = lang_progress.get("status", "pending")
|
||||
|
||||
if status == "failed":
|
||||
_any_failed = True # noqa: F841 - reserved for future use
|
||||
elif status != "completed":
|
||||
all_complete = False
|
||||
any_failed = True
|
||||
failed_languages.append({
|
||||
"language": language,
|
||||
"error": lang_progress.get("error_message", "Unknown error")
|
||||
})
|
||||
elif status not in ["completed", "failed"]:
|
||||
# Still pending or rendering
|
||||
all_finished = False
|
||||
|
||||
if all_complete:
|
||||
logger.info(f"All accessible videos complete for job {job_id}")
|
||||
job_title = job_doc.get("title", "Untitled Job")
|
||||
|
||||
# If job is still in TTS_GENERATING or RENDERING_VIDEO, transition to PENDING_FINAL_REVIEW
|
||||
if job_doc["status"] in [JobStatus.TTS_GENERATING.value, JobStatus.RENDERING_VIDEO.value]:
|
||||
await db.jobs.update_one(
|
||||
{"_id": job_id},
|
||||
{
|
||||
"$set": {
|
||||
"status": JobStatus.PENDING_FINAL_REVIEW.value,
|
||||
"updated_at": datetime.utcnow()
|
||||
},
|
||||
"$push": {
|
||||
"review.history": {
|
||||
"at": datetime.utcnow(),
|
||||
"status": JobStatus.PENDING_FINAL_REVIEW.value,
|
||||
"by": "system"
|
||||
# Only update job status if all languages have finished processing
|
||||
if all_finished:
|
||||
if any_failed:
|
||||
# Some or all videos failed - transition to RENDER_FAILED
|
||||
logger.error(f"Accessible video rendering failed for job {job_id}: {len(failed_languages)} language(s) failed")
|
||||
|
||||
# Build error summary
|
||||
error_summary = "; ".join([f"{f['language']}: {f['error']}" for f in failed_languages])
|
||||
|
||||
if job_doc["status"] in [JobStatus.TTS_GENERATING.value, JobStatus.RENDERING_VIDEO.value]:
|
||||
await db.jobs.update_one(
|
||||
{"_id": job_id},
|
||||
{
|
||||
"$set": {
|
||||
"status": JobStatus.RENDER_FAILED.value,
|
||||
"error": {
|
||||
"type": "render_failure",
|
||||
"failed_languages": failed_languages,
|
||||
"message": f"Video rendering failed for {len(failed_languages)} language(s): {error_summary}",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
},
|
||||
"updated_at": datetime.utcnow()
|
||||
},
|
||||
"$push": {
|
||||
"review.history": {
|
||||
"at": datetime.utcnow(),
|
||||
"status": JobStatus.RENDER_FAILED.value,
|
||||
"by": "system",
|
||||
"notes": f"Rendering failed for: {', '.join([f['language'] for f in failed_languages])}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
job_title = job_doc.get("title", "Untitled Job")
|
||||
broadcast_status_update(
|
||||
job_id,
|
||||
JobStatus.PENDING_FINAL_REVIEW.value,
|
||||
job_title=job_title,
|
||||
message=f"{job_title} has all accessible videos complete - ready for Final Review"
|
||||
)
|
||||
broadcast_status_update(
|
||||
job_id,
|
||||
JobStatus.RENDER_FAILED.value,
|
||||
job_title=job_title,
|
||||
message=f"{job_title} - Video rendering failed for {len(failed_languages)} language(s). Manual reprocessing required."
|
||||
)
|
||||
else:
|
||||
# All videos completed successfully
|
||||
logger.info(f"All accessible videos complete for job {job_id}")
|
||||
|
||||
if job_doc["status"] in [JobStatus.TTS_GENERATING.value, JobStatus.RENDERING_VIDEO.value]:
|
||||
await db.jobs.update_one(
|
||||
{"_id": job_id},
|
||||
{
|
||||
"$set": {
|
||||
"status": JobStatus.PENDING_FINAL_REVIEW.value,
|
||||
"updated_at": datetime.utcnow()
|
||||
},
|
||||
"$push": {
|
||||
"review.history": {
|
||||
"at": datetime.utcnow(),
|
||||
"status": JobStatus.PENDING_FINAL_REVIEW.value,
|
||||
"by": "system"
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
broadcast_status_update(
|
||||
job_id,
|
||||
JobStatus.PENDING_FINAL_REVIEW.value,
|
||||
job_title=job_title,
|
||||
message=f"{job_title} has all accessible videos complete - ready for Final Review"
|
||||
)
|
||||
|
||||
|
||||
async def _refine_pause_points_with_whisper(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ export function StatusBadge({ status }: StatusBadgeProps) {
|
|||
return 'bg-green-100 text-green-800';
|
||||
case 'rejected':
|
||||
case 'tts_failed':
|
||||
case 'render_failed':
|
||||
return 'bg-red-100 text-red-800';
|
||||
case 'translating':
|
||||
return 'bg-blue-100 text-blue-800';
|
||||
|
|
@ -54,6 +55,8 @@ export function StatusBadge({ status }: StatusBadgeProps) {
|
|||
return 'Rejected';
|
||||
case 'tts_failed':
|
||||
return 'TTS Failed';
|
||||
case 'render_failed':
|
||||
return 'Render Failed';
|
||||
case 'translating':
|
||||
return 'Translating';
|
||||
case 'tts_generating':
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ const ProgressIndicator = ({ status }: { status: string }) => {
|
|||
'approved', // Generic - matches both approved_english and approved_source
|
||||
'translating',
|
||||
'tts_generating',
|
||||
'rendering_video',
|
||||
'pending_final_review',
|
||||
'completed'
|
||||
];
|
||||
|
|
@ -24,6 +25,12 @@ const ProgressIndicator = ({ status }: { status: string }) => {
|
|||
const normalizedStatus = (status === 'approved_english' || status === 'approved_source') ? 'approved' : status;
|
||||
const currentIndex = steps.indexOf(normalizedStatus);
|
||||
const isRejected = status === 'rejected';
|
||||
const isFailed = status === 'tts_failed' || status === 'render_failed';
|
||||
|
||||
// For failed states, find which step they failed at
|
||||
const failedAtStep = status === 'tts_failed' ? 'tts_generating' :
|
||||
status === 'render_failed' ? 'rendering_video' : null;
|
||||
const failedAtIndex = failedAtStep ? steps.indexOf(failedAtStep) : -1;
|
||||
|
||||
if (isRejected) {
|
||||
return (
|
||||
|
|
@ -36,6 +43,37 @@ const ProgressIndicator = ({ status }: { status: string }) => {
|
|||
);
|
||||
}
|
||||
|
||||
if (isFailed) {
|
||||
const failedStepLabel = failedAtStep?.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
|
||||
return (
|
||||
<div className="bg-white border border-gray-200 rounded-lg p-4">
|
||||
<h3 className="text-sm font-medium text-gray-900 mb-3">Processing Progress</h3>
|
||||
<div className="space-y-2">
|
||||
{steps.map((step, index) => (
|
||||
<div key={step} className="flex items-center">
|
||||
<div className={`w-3 h-3 rounded-full mr-3 ${
|
||||
index < failedAtIndex ? 'bg-green-500' :
|
||||
index === failedAtIndex ? 'bg-red-500' : 'bg-gray-300'
|
||||
}`} />
|
||||
<span className={`text-sm ${
|
||||
index === failedAtIndex ? 'font-medium text-red-600' :
|
||||
index < failedAtIndex ? 'text-gray-600' : 'text-gray-400'
|
||||
}`}>
|
||||
{step.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())}
|
||||
{index === failedAtIndex && ' (Failed)'}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div className="mt-3 pt-3 border-t border-gray-200">
|
||||
<p className="text-xs text-red-600">
|
||||
Processing failed at {failedStepLabel}. Check error details for more information.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="bg-white border border-gray-200 rounded-lg p-4">
|
||||
<h3 className="text-sm font-medium text-gray-900 mb-3">Processing Progress</h3>
|
||||
|
|
@ -46,7 +84,7 @@ const ProgressIndicator = ({ status }: { status: string }) => {
|
|||
index <= currentIndex ? 'bg-green-500' : 'bg-gray-300'
|
||||
}`} />
|
||||
<span className={`text-sm ${
|
||||
index === currentIndex ? 'font-medium text-gray-900' :
|
||||
index === currentIndex ? 'font-medium text-gray-900' :
|
||||
index < currentIndex ? 'text-gray-600' : 'text-gray-400'
|
||||
}`}>
|
||||
{step.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())}
|
||||
|
|
@ -397,8 +435,38 @@ export function JobDetail() {
|
|||
{/* Error Display */}
|
||||
{job.error && (
|
||||
<div className="bg-red-50 border border-red-200 rounded-lg p-4">
|
||||
<h3 className="text-sm font-medium text-red-800 mb-2">Processing Error</h3>
|
||||
<p className="text-xs text-red-600">{String(job.error.message || 'Unknown error')}</p>
|
||||
<div className="flex items-start gap-2 mb-2">
|
||||
<svg className="w-5 h-5 text-red-600 flex-shrink-0 mt-0.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
|
||||
</svg>
|
||||
<h3 className="text-sm font-medium text-red-800">Processing Error</h3>
|
||||
</div>
|
||||
<p className="text-sm text-red-700 mb-2">
|
||||
{String(job.error.message || 'An error occurred during processing')}
|
||||
</p>
|
||||
{job.error.type && (
|
||||
<p className="text-xs text-red-600 mb-2">
|
||||
<span className="font-medium">Type:</span> {String(job.error.type).replace(/_/g, ' ')}
|
||||
</p>
|
||||
)}
|
||||
{/* Show failed languages for render failures */}
|
||||
{job.error.failed_languages && Array.isArray(job.error.failed_languages) && (
|
||||
<div className="mt-3 pt-3 border-t border-red-200">
|
||||
<p className="text-xs font-medium text-red-800 mb-2">Failed Languages:</p>
|
||||
<ul className="space-y-1">
|
||||
{(job.error.failed_languages as Array<{language: string; error: string}>).map((item, idx) => (
|
||||
<li key={idx} className="text-xs text-red-600">
|
||||
<span className="font-medium">{item.language.toUpperCase()}:</span> {item.error}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
{job.error.timestamp && (
|
||||
<p className="text-xs text-red-500 mt-2">
|
||||
{formatDistanceToNow(new Date(String(job.error.timestamp)))} ago
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -24,7 +24,9 @@ const STATUS_LABELS: Record<string, string> = {
|
|||
'qc_feedback': 'QC Feedback',
|
||||
'translating': 'Translating',
|
||||
'tts_generating': 'Generating Audio',
|
||||
'tts_failed': 'TTS Failed',
|
||||
'rendering_video': 'Rendering Video',
|
||||
'render_failed': 'Render Failed',
|
||||
'pending_final_review': 'Pending Final Review',
|
||||
'completed': 'Completed',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ export type JobStatus =
|
|||
| "tts_generating"
|
||||
| "tts_failed" // TTS synthesis failed after retries, requires reprocessing
|
||||
| "rendering_video" // Accessible video rendering in progress
|
||||
| "render_failed" // Accessible video rendering failed, requires reprocessing
|
||||
| "pending_final_review"
|
||||
| "completed";
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue