diff --git a/backend/.env.development b/backend/.env.development index c0d92f9..bc5f287 100644 --- a/backend/.env.development +++ b/backend/.env.development @@ -11,4 +11,10 @@ FLASK_DEBUG=True PORT=7394 # CORS Configuration - Allow localhost for development -FRONTEND_URL=http://localhost:3000 \ No newline at end of file +FRONTEND_URL=http://localhost:3000 + +# File Cleanup Configuration (optimized for development - faster cleanup) +CLEANUP_DELAY_SMALL_FILES=10 +CLEANUP_DELAY_LARGE_FILES=20 +LARGE_FILE_THRESHOLD=50 +CLEANUP_GCS_BY_DEFAULT=false \ No newline at end of file diff --git a/backend/config.py b/backend/config.py index 7d1d9ec..1993757 100644 --- a/backend/config.py +++ b/backend/config.py @@ -55,4 +55,10 @@ class Config: # Webhook Configuration WEBHOOK_URL = os.getenv('WEBHOOK_URL', 'https://hook.us1.make.celonis.com/8ri1h8b2he4wudp2jku69mgcxumzxf3v') WEBHOOK_ENABLED = os.getenv('WEBHOOK_ENABLED', 'true').lower() in ['true', '1', 'yes'] - WEBHOOK_TIMEOUT = int(os.getenv('WEBHOOK_TIMEOUT', 10)) # seconds \ No newline at end of file + WEBHOOK_TIMEOUT = int(os.getenv('WEBHOOK_TIMEOUT', 10)) # seconds + + # File Cleanup Configuration + CLEANUP_DELAY_SMALL_FILES = int(os.getenv('CLEANUP_DELAY_SMALL_FILES', 15)) # seconds + CLEANUP_DELAY_LARGE_FILES = int(os.getenv('CLEANUP_DELAY_LARGE_FILES', 30)) # seconds + LARGE_FILE_THRESHOLD = int(os.getenv('LARGE_FILE_THRESHOLD', 100)) * 1024 * 1024 # MB to bytes + CLEANUP_GCS_BY_DEFAULT = os.getenv('CLEANUP_GCS_BY_DEFAULT', 'true').lower() in ['true', '1', 'yes'] \ No newline at end of file diff --git a/backend/routes/api.py b/backend/routes/api.py index 67cb81b..6452a97 100644 --- a/backend/routes/api.py +++ b/backend/routes/api.py @@ -286,14 +286,30 @@ def download_video(job_id): if not video_path or not os.path.exists(video_path): return jsonify({'error': f'Video file not found at path: {video_path}'}), 404 - # Re-enable cleanup with longer delay for zip files to ensure download completes + # Smart cleanup: delay based on file size and user configuration def cleanup_after_send(): - time.sleep(5) # Additional delay to ensure send_file completes - cleanup_job_files(job_id) + # Brief delay to ensure download starts + time.sleep(2) + + # Use configurable cleanup (GCS cleanup can be disabled for faster cleanup) + cleanup_job_files(job_id, cleanup_gcs=Config.CLEANUP_GCS_BY_DEFAULT) - cleanup_thread = threading.Timer(60.0, cleanup_after_send) # 60 second delay + # Determine cleanup delay based on file size + file_size = 0 + if os.path.exists(video_path): + file_size = os.path.getsize(video_path) + + if file_size > Config.LARGE_FILE_THRESHOLD: + cleanup_delay = Config.CLEANUP_DELAY_LARGE_FILES + else: + cleanup_delay = Config.CLEANUP_DELAY_SMALL_FILES + + cleanup_thread = threading.Timer(cleanup_delay, cleanup_after_send) cleanup_thread.start() + cleanup_type = "full (local + GCS)" if Config.CLEANUP_GCS_BY_DEFAULT else "local only" + print(f"Scheduled {cleanup_type} cleanup in {cleanup_delay}s for job {job_id} (file size: {file_size / (1024*1024):.1f}MB)") + # Determine download type and filename is_zip = job.get('is_zip', False) download_type = job.get('download_type', 'zip') @@ -400,9 +416,9 @@ def download_all_videos(job_id): @api_bp.route('/cleanup/', methods=['DELETE']) def cleanup_job(job_id): - """Clean up job files manually.""" + """Clean up job files manually (full cleanup including GCS).""" try: - success = cleanup_job_files(job_id) + success = cleanup_job_files(job_id, cleanup_gcs=True) if success: return jsonify({'message': 'Files cleaned up successfully'}), 200 else: @@ -410,6 +426,18 @@ def cleanup_job(job_id): except Exception as e: return jsonify({'error': f'Failed to cleanup: {str(e)}'}), 500 +@api_bp.route('/cleanup//local', methods=['DELETE']) +def cleanup_job_local_only(job_id): + """Fast cleanup of local files only (keeps GCS files for potential re-download).""" + try: + success = cleanup_job_files(job_id, cleanup_gcs=False) + if success: + return jsonify({'message': 'Local files cleaned up successfully (GCS files preserved)'}), 200 + else: + return jsonify({'error': 'Failed to clean up some local files'}), 500 + except Exception as e: + return jsonify({'error': f'Failed to cleanup local files: {str(e)}'}), 500 + @api_bp.route('/user-jobs', methods=['GET']) def get_user_job_list(): """Get all jobs for the current user.""" diff --git a/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_1.mp4 b/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_1.mp4 deleted file mode 100644 index 1fa0a2e..0000000 Binary files a/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_1.mp4 and /dev/null differ diff --git a/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_2.mp4 b/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_2.mp4 deleted file mode 100644 index 8e49eed..0000000 Binary files a/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_2.mp4 and /dev/null differ diff --git a/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_3.mp4 b/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_3.mp4 deleted file mode 100644 index 1544f31..0000000 Binary files a/backend/temp_downloads/job_485524df-1d69-4dc0-8970-4952c397077b/generated_video_3.mp4 and /dev/null differ diff --git a/backend/video_generator.py b/backend/video_generator.py index c461b71..7ce42bd 100644 --- a/backend/video_generator.py +++ b/backend/video_generator.py @@ -841,14 +841,21 @@ def get_queue_status() -> dict: 'active_jobs': processing_jobs.copy() } -def cleanup_job_files(job_id: str) -> bool: - """Clean up local and GCS files for a job.""" +def cleanup_job_files(job_id: str, cleanup_gcs: bool = True) -> bool: + """ + Clean up local and optionally GCS files for a job. + + Args: + job_id: The job ID to clean up + cleanup_gcs: Whether to also clean up GCS files (default: True) + """ job = job_status.get(job_id) if not job: print(f"Job {job_id} not found for cleanup") return False success = True + start_time = time.time() # Clean up entire job folder (more efficient than individual files) download_folder = job.get('download_folder') @@ -881,30 +888,34 @@ def cleanup_job_files(job_id: str) -> bool: print(f"Error deleting individual video file: {e}") success = False - # Delete GCS video files (for backwards compatibility, keep the old logic) - if job.get('source_bucket_name') and job.get('source_blob_name'): - try: - delete_blob(job['source_bucket_name'], job['source_blob_name']) - except Exception as e: - print(f"Error deleting GCS file: {e}") - success = False - - # Delete multiple GCS video files - if job.get('gcs_video_uris'): - for gcs_uri in job['gcs_video_uris']: + # Delete GCS video files only if requested (can be slow) + if cleanup_gcs: + # Delete GCS video files (for backwards compatibility, keep the old logic) + if job.get('source_bucket_name') and job.get('source_blob_name'): try: - if gcs_uri.startswith("gs://"): - path_parts = gcs_uri.replace("gs://", "").split("/", 1) - bucket_name = path_parts[0] - blob_name = path_parts[1] - delete_blob(bucket_name, blob_name) - print(f"Deleted GCS file: {gcs_uri}") + delete_blob(job['source_bucket_name'], job['source_blob_name']) except Exception as e: - print(f"Error deleting GCS file {gcs_uri}: {e}") + print(f"Error deleting GCS file: {e}") success = False + + # Delete multiple GCS video files + if job.get('gcs_video_uris'): + for gcs_uri in job['gcs_video_uris']: + try: + if gcs_uri.startswith("gs://"): + path_parts = gcs_uri.replace("gs://", "").split("/", 1) + bucket_name = path_parts[0] + blob_name = path_parts[1] + delete_blob(bucket_name, blob_name) + print(f"Deleted GCS file: {gcs_uri}") + except Exception as e: + print(f"Error deleting GCS file {gcs_uri}: {e}") + success = False + else: + print("Skipping GCS cleanup (local files only)") - # Delete image files - if job.get('local_image_path') or job.get('image_blob_name'): + # Delete image files (includes both local and GCS) + if cleanup_gcs and (job.get('local_image_path') or job.get('image_blob_name')): try: cleanup_image_files( job_id, @@ -919,4 +930,9 @@ def cleanup_job_files(job_id: str) -> bool: if success: del job_status[job_id] + # Log cleanup completion time + cleanup_time = time.time() - start_time + cleanup_type = "full (local + GCS)" if cleanup_gcs else "local only" + print(f"Cleanup completed for job {job_id} in {cleanup_time:.2f}s ({cleanup_type})") + return success \ No newline at end of file diff --git a/frontend/src/components/ProgressIndicator.jsx b/frontend/src/components/ProgressIndicator.jsx index e692083..bdbfaa4 100644 --- a/frontend/src/components/ProgressIndicator.jsx +++ b/frontend/src/components/ProgressIndicator.jsx @@ -96,19 +96,43 @@ const ProgressIndicator = ({ )} {status === JOB_STATUS.COMPLETED && ( - - - - Your content is ready for download! - - + + + + + Your content is ready for download! + + + + + {/* Show additional download links if multiple videos */} + {downloadLinks && downloadLinks.length > 1 && ( + + + Individual Downloads: + + + {downloadLinks.slice(1).map((link, index) => ( + + ))} + + + )} )} diff --git a/frontend/src/hooks/useVideoGeneration.js b/frontend/src/hooks/useVideoGeneration.js index 4aea7de..c59eaee 100644 --- a/frontend/src/hooks/useVideoGeneration.js +++ b/frontend/src/hooks/useVideoGeneration.js @@ -1,6 +1,6 @@ import { useState } from 'react'; import { generateVideo, checkJobStatus, downloadVideo } from '../services/api'; -import { JOB_STATUS } from '../utils/constants'; +import { JOB_STATUS, API_BASE_URL } from '../utils/constants'; export const useVideoGeneration = () => { const [isGenerating, setIsGenerating] = useState(false); @@ -35,13 +35,37 @@ export const useVideoGeneration = () => { try { const statusResponse = await checkJobStatus(jobId); + console.log('Status update:', statusResponse); + setStatus(statusResponse.status); setProgress(statusResponse.progress || 0); setMessage(statusResponse.message || ''); if (statusResponse.status === JOB_STATUS.COMPLETED) { setIsGenerating(false); - setDownloadLinks(statusResponse.download_links || []); + + // Build download links from the actual backend response structure + const links = []; + if (statusResponse.video_path) { + links.push({ + url: `${API_BASE_URL}/api/download/${jobId}`, + label: statusResponse.video_count > 1 ? 'Download All' : 'Download Video', + type: statusResponse.download_type || 'video' + }); + } + + // Add individual video download links + if (statusResponse.video_count > 1) { + for (let i = 1; i <= statusResponse.video_count; i++) { + links.push({ + url: `${API_BASE_URL}/api/download/${jobId}/video/${i}`, + label: `Video ${i}`, + type: 'video' + }); + } + } + + setDownloadLinks(links); return; } @@ -51,9 +75,34 @@ export const useVideoGeneration = () => { return; } + if (statusResponse.status === JOB_STATUS.CANCELLED) { + setError('Job was cancelled'); + setIsGenerating(false); + return; + } + // Continue polling if still in progress - if ([JOB_STATUS.STARTING, JOB_STATUS.GENERATING, JOB_STATUS.PROCESSING, JOB_STATUS.DOWNLOADING].includes(statusResponse.status)) { - setTimeout(() => pollStatus(jobId), 3000); // Poll every 3 seconds + const activeStatuses = [ + JOB_STATUS.QUEUED, + JOB_STATUS.STARTING, + JOB_STATUS.UPLOADING_IMAGE, + JOB_STATUS.GENERATING, + JOB_STATUS.PROCESSING, + JOB_STATUS.DOWNLOADING, + JOB_STATUS.RETRY_1_OF_3, + JOB_STATUS.RETRY_2_OF_3, + JOB_STATUS.RETRY_3_OF_3 + ]; + + if (activeStatuses.includes(statusResponse.status)) { + console.log(`Continuing to poll. Status: ${statusResponse.status}, Progress: ${statusResponse.progress}%`); + setTimeout(() => pollStatus(jobId), 2000); // Poll every 2 seconds for better responsiveness + } else { + // Handle unexpected status by stopping polling and showing error + console.error('Unknown status received:', statusResponse.status, 'Full response:', statusResponse); + console.error('Active statuses:', activeStatuses); + setError(`Unknown status: ${statusResponse.status}`); + setIsGenerating(false); } } catch (err) { setError(err.response?.data?.error || err.message || 'Failed to check status');