From 3008d8f8fc9670ce5dc2c5ba301196e552d6a10a Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 18 Sep 2025 14:25:24 -0500 Subject: [PATCH] initial commit --- .gitignore | 384 + CLAUDE.md | 20 + DEPLOYMENT.md | 105 + LOG_EXTRACTION_README.md | 111 + README.md | 143 + backend/.env | 1 + backend/README.md | 41 + backend/app.py | 1112 + backend/auth.py | 198 + backend/chunked_upload.py | 166 + backend/fix_jose.sh | 24 + backend/install_wkhtmltopdf.sh | 50 + backend/requirements-py310.txt | 10 + backend/requirements.txt | 55 + backend/run.py | 40 + backend/test_api.py | 23 + backend/test_webhook.py | 137 + backend/test_webhook_manual.py | 46 + backend/video-query.service | 17 + backend/video_processor.py | 224 + extract_user_logs.sh | 66 + extract_user_logs_robust.sh | 175 + frontend/.env | 2 + frontend/README.md | 54 + frontend/build.sh | 18 + frontend/package-lock.json | 18770 ++++++++++++++++ frontend/package.json | 40 + frontend/public/.htaccess | 10 + frontend/public/index.html | 23 + frontend/public/manifest.json | 25 + frontend/requirements.txt | 48 + frontend/src/App.js | 388 + frontend/src/auth/AuthProvider.js | 167 + frontend/src/auth/authApiClient.js | 146 + frontend/src/auth/authConfig.js | 44 + .../src/components/AuthenticatedContent.js | 99 + frontend/src/components/Login.js | 137 + frontend/src/components/PromptSelector.js | 53 + frontend/src/components/ResultDisplay.js | 401 + frontend/src/components/VideoUpload.js | 75 + frontend/src/index.css | 213 + frontend/src/index.js | 15 + frontend/src/utils/chunkedUploader.js | 210 + quick_extract.sh | 17 + requirements.txt | 43 + restart.sh | 26 + video_query.py | 141 + 47 files changed, 24313 insertions(+) create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 DEPLOYMENT.md create mode 100644 LOG_EXTRACTION_README.md create mode 100644 README.md create mode 100644 backend/.env create mode 100644 backend/README.md create mode 100644 backend/app.py create mode 100644 backend/auth.py create mode 100644 backend/chunked_upload.py create mode 100755 backend/fix_jose.sh create mode 100755 backend/install_wkhtmltopdf.sh create mode 100644 backend/requirements-py310.txt create mode 100644 backend/requirements.txt create mode 100755 backend/run.py create mode 100644 backend/test_api.py create mode 100644 backend/test_webhook.py create mode 100644 backend/test_webhook_manual.py create mode 100644 backend/video-query.service create mode 100644 backend/video_processor.py create mode 100755 extract_user_logs.sh create mode 100755 extract_user_logs_robust.sh create mode 100644 frontend/.env create mode 100644 frontend/README.md create mode 100755 frontend/build.sh create mode 100644 frontend/package-lock.json create mode 100644 frontend/package.json create mode 100644 frontend/public/.htaccess create mode 100644 frontend/public/index.html create mode 100644 frontend/public/manifest.json create mode 100644 frontend/requirements.txt create mode 100644 frontend/src/App.js create mode 100644 frontend/src/auth/AuthProvider.js create mode 100644 frontend/src/auth/authApiClient.js create mode 100644 frontend/src/auth/authConfig.js create mode 100644 frontend/src/components/AuthenticatedContent.js create mode 100644 frontend/src/components/Login.js create mode 100644 frontend/src/components/PromptSelector.js create mode 100644 frontend/src/components/ResultDisplay.js create mode 100644 frontend/src/components/VideoUpload.js create mode 100644 frontend/src/index.css create mode 100644 frontend/src/index.js create mode 100644 frontend/src/utils/chunkedUploader.js create mode 100755 quick_extract.sh create mode 100644 requirements.txt create mode 100755 restart.sh create mode 100644 video_query.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..66b94b0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,384 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be added to the global gitignore or merged into this project gitignore. For a PyCharm +# project, it is recommended to ignore the entire .idea directory. +.idea/ + +# Node.js (for frontend) +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage/ +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# Storybook build outputs +.out +.storybook-out +storybook-static + +# Rollup.js default build output +dist/ + +# Uncomment the public line if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# macOS +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Windows +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Linux +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +# Project specific +*.png +*.jpg +*.jpeg +*.gif +*.mp4 +*.avi +*.mov +*.mkv +*.webm + +# Logs +*.log +logs/ + +# IDE +*.swp +*.swo +*~ + +# Temporary files +.tmp/ +temp/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..f0e10f6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,20 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build/Lint/Test Commands +- Run script: `python video_query.py [--prompt "Your custom prompt"]` +- Install dependencies: `pip install -r requirements.txt` (if requirements.txt exists) +- Create venv: `python -m venv venv && source venv/bin/activate` +- Install required packages: `pip install google-generativeai` + +## Code Style Guidelines +- Imports: Standard library imports first, followed by third-party imports, then local imports +- Formatting: PEP 8 compliant with 4-space indentation +- Types: Use type hints for function parameters and return values +- Naming: snake_case for variables/functions, PascalCase for classes +- Error handling: Use try/except blocks with specific exception types +- API Keys: Store in environment variables, not hardcoded +- Documentation: Use docstrings for functions and main modules +- Max line length: 100 characters +- Include helpful comments for complex operations \ No newline at end of file diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..bfc7792 --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,105 @@ +# Deployment Instructions for PDF Generation Feature + +This document outlines the steps needed to deploy the new PDF generation feature to your production server. + +## Prerequisites + +The PDF generation functionality requires several system packages to be installed on the server where the backend runs: + +### For Ubuntu/Debian: + +```bash +sudo apt-get update +sudo apt-get install -y wkhtmltopdf python3-cairo python3-pil libcairo2-dev +``` + +### For CentOS/RHEL: + +```bash +sudo yum install -y wkhtmltopdf cairo-devel pango-devel libffi-devel python3-pillow +``` + +## Deployment Steps + +### 1. Backend Deployment + +1. **Install Required Python Packages**: + + ```bash + pip install pdfkit pillow cairosvg + ``` + +2. **Update Backend Requirements**: + + Upload the updated `requirements.txt` to your server to ensure all new dependencies are documented. + +3. **Upload Backend Files**: + + Copy the updated `app.py` file to your server. + +### 2. Frontend Deployment + +1. **Build the Frontend**: + + ```bash + cd /path/to/video_query/frontend + npm run build + ``` + +2. **Deploy the Built Files**: + + Copy the entire contents of the `build` directory to your web server's hosting location: + + ```bash + scp -r build/* user@your-server:/path/to/webroot/video_query/ + ``` + +## Testing the Deployment + +1. **Test PDF Generation**: + - Process a video and once results are displayed, click the "Download PDF" button + - Verify that a well-formatted PDF is downloaded that includes any mermaid diagrams if present + +2. **Check Server Logs**: + - Monitor the backend logs to ensure PDF generation is working properly: + ```bash + sudo journalctl -u video-query -f + ``` + +## Troubleshooting + +If PDF generation fails, check the following: + +1. **wkhtmltopdf Installation**: + - Verify wkhtmltopdf is properly installed: + ```bash + wkhtmltopdf --version + ``` + +2. **File Permissions**: + - Ensure the application has write permissions to the temporary directory: + ```bash + sudo chown -R app-user:app-user /tmp + ``` + +3. **Network Issues**: + - If there are CORS errors, verify that the CORS configuration in `app.py` includes the PDF endpoint. + +## Reverting the Changes + +If you need to revert: + +1. **Backend**: + - Restore the previous version of `app.py` from your backup + - Restart the backend service: + ```bash + sudo systemctl restart video-query + ``` + +2. **Frontend**: + - Redeploy the previous version of the frontend build + +## Support + +If you encounter any issues with the deployment, please refer to the pdfkit documentation at: +https://pypi.org/project/pdfkit/ \ No newline at end of file diff --git a/LOG_EXTRACTION_README.md b/LOG_EXTRACTION_README.md new file mode 100644 index 0000000..4f5e225 --- /dev/null +++ b/LOG_EXTRACTION_README.md @@ -0,0 +1,111 @@ +# Log Extraction Scripts + +These scripts extract user email addresses and prompts from systemd logs for the video generation service. + +## Scripts Overview + +### 1. `extract_user_logs.sh` - Basic Extraction +Simple script that extracts basic information and creates a CSV report. + +**Usage:** +```bash +./extract_user_logs.sh [output_file.csv] +``` + +**Examples:** +```bash +# Use default output file (video_generation_usage.csv) +./extract_user_logs.sh + +# Specify custom output file +./extract_user_logs.sh my_usage_report.csv +``` + +### 2. `extract_user_logs_robust.sh` - Enhanced Extraction +More robust script with error handling, progress indicators, and detailed reporting. + +**Usage:** +```bash +./extract_user_logs_robust.sh [output_file.csv] [service_name] [date_range] +``` + +**Examples:** +```bash +# Basic usage +./extract_user_logs_robust.sh + +# Custom output file +./extract_user_logs_robust.sh usage_report.csv + +# Different service name +./extract_user_logs_robust.sh report.csv my-video-service + +# Specific date range (last 30 days) +./extract_user_logs_robust.sh report.csv veo-video-generator "--since=30 days ago" + +# Specific date range (from June 1st, 2024) +./extract_user_logs_robust.sh report.csv veo-video-generator "--since=2024-06-01" + +# Between specific dates +./extract_user_logs_robust.sh report.csv veo-video-generator "--since=2024-06-01 --until=2024-06-15" +``` + +### 3. `quick_extract.sh` - Quick & Simple +One-liner style extraction for quick checks. + +**Usage:** +```bash +./quick_extract.sh [service_name] +``` + +## Requirements + +- `jq` - JSON processor (install with: `sudo apt install jq` or `brew install jq`) +- `journalctl` - systemd journal viewer (usually pre-installed on systemd systems) +- Bash shell + +## Output Format + +The scripts generate CSV files with the following columns: + +- `timestamp` - When the request was made +- `user_email` - Email address of the user +- `prompt` - The generation prompt used +- `video_length_sec` - Length of video requested (seconds) +- `aspect_ratio` - Video aspect ratio +- `person_generation` - Person generation setting + +## Log Format Expected + +The scripts look for log lines matching this pattern: +``` +Jun 14 20:23:20 optical-web-1 veo-video-generator[3458795]: DEBUG: Raw JSON data received: {'prompt': '...', 'user_email': '...', ...} +``` + +## Troubleshooting + +### No records found +- Check if the service name is correct: `systemctl list-units | grep video` +- Verify logs exist: `journalctl -u veo-video-generator | head` +- Check if the log format matches what the script expects + +### Permission denied +- You may need to run with sudo to access systemd logs: `sudo ./extract_user_logs.sh` + +### jq command not found +- Install jq: + - Ubuntu/Debian: `sudo apt install jq` + - macOS: `brew install jq` + - CentOS/RHEL: `sudo yum install jq` + +### Invalid JSON errors +- The robust script will show warnings for malformed JSON +- Check the log format to ensure it matches the expected pattern + +## Example Output + +```csv +timestamp,user_email,prompt,video_length_sec,aspect_ratio,person_generation +"2024-06-14T20:23:20","MichaelClervi@oliver.agency","three adults telling jokes on the beach at sunset","8","16:9","allow_adult" +"2024-06-14T21:15:30","user@example.com","a cat playing piano","5","16:9","allow_adult" +``` \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..61311a2 --- /dev/null +++ b/README.md @@ -0,0 +1,143 @@ +# Video Query Tool + +This application processes videos using Google's Gemini AI model, allowing users to: + +1. Upload videos (MP4, AVI, MOV, etc.) +2. Choose from preset processing modes or use custom prompts +3. Get AI-generated markdown content based on the video content + +## Important Notes + +- **Video Length Limitation**: The Gemini AI model can only process videos up to 55 minutes in length. +- **File Size**: The application supports uploads up to 5GB. + +## Project Structure + +``` +video_query/ +├── backend/ # Flask/Hypercorn server +│ ├── app.py # Main Flask application +│ ├── video_processor.py # Video processing logic +│ └── run.py # Hypercorn server script +└── frontend/ # React frontend + ├── public/ # Static assets + └── src/ # React source code +``` + +## Setup Instructions + +### Backend Setup + +1. Create and activate a virtual environment: + ``` + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + ``` + +2. Install backend dependencies: + ``` + pip install -r requirements.txt + ``` + +3. Set your Google API key: + ``` + export GOOGLE_API_KEY=your_api_key_here + ``` + +4. Run the development server: + ``` + cd backend + python run.py + ``` + +### Frontend Setup + +1. Install Node.js dependencies: + ``` + cd frontend + npm install + ``` + +2. Start the development server: + ``` + npm start + ``` + +## Deployment + +### Backend Deployment with Systemd + +1. Update the systemd service file (`backend/video-query.service`): + - Update paths to match your server + - Add your GOOGLE_API_KEY + - Place in `/etc/systemd/system/` + +2. Enable and start the service: + ``` + sudo systemctl enable video-query + sudo systemctl start video-query + ``` + +3. Check the service status: + ``` + sudo systemctl status video-query + ``` + +### Frontend Deployment with Apache + +1. Build the React frontend: + ``` + cd frontend + npm run build + ``` + +2. Copy the build directory to your Apache document root: + ``` + cp -r build/* /var/www/html/video-query/ + ``` + +3. Configure Apache to serve the React app, adding the following to your Apache configuration: + ``` + + ServerName yourdomain.com + DocumentRoot /var/www/html/video-query + + + AllowOverride All + Require all granted + + # Redirect all requests to index.html for React routing + RewriteEngine On + RewriteBase / + RewriteRule ^index\.html$ - [L] + RewriteCond %{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_FILENAME} !-d + RewriteRule . /index.html [L] + + + # Proxy API requests to the backend + ProxyPass /api http://localhost:5010/api + ProxyPassReverse /api http://localhost:5010/api + + ``` + +4. Restart Apache: + ``` + sudo systemctl restart apache2 + ``` + +## API Reference + +The backend API exposes a single endpoint: + +- **POST /api/process**: Processes an uploaded video with the specified prompt + - Form parameters: + - `video`: The video file + - `prompt`: The prompt text to process the video with + - Returns: + - Success: `{ "success": true, "content": "markdown content..." }` + - Error: `{ "success": false, "message": "error message..." }` + +## License + +This project is proprietary and confidential. \ No newline at end of file diff --git a/backend/.env b/backend/.env new file mode 100644 index 0000000..423528e --- /dev/null +++ b/backend/.env @@ -0,0 +1 @@ +GOOGLE_API_KEY=AIzaSyBF3Ia1nVS4PLuLpWt-85ct_heJ7FrlvkQ \ No newline at end of file diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..c4c31fb --- /dev/null +++ b/backend/README.md @@ -0,0 +1,41 @@ +# Video Query Backend Service + +This backend service processes videos using Google's Gemini API. + +## Installation for Python 3.10 (Server) + +If you're running on Python 3.10, use these installation instructions: + +```bash +# Create and activate virtual environment +python -m venv venv +source venv/bin/activate + +# Install dependencies +pip install -r requirements-py310.txt + +# Fix potential jose module conflict +bash fix_jose.sh +``` + +## Running the Service + +```bash +python run.py +``` + +## Troubleshooting + +If you encounter a SyntaxError related to the `jose` module: + +``` +SyntaxError: Missing parentheses in call to 'print'. Did you mean print(...)? +``` + +Run the provided fix script: + +```bash +bash fix_jose.sh +``` + +This script will properly uninstall any conflicting jose modules and install the correct python-jose package. \ No newline at end of file diff --git a/backend/app.py b/backend/app.py new file mode 100644 index 0000000..90359b5 --- /dev/null +++ b/backend/app.py @@ -0,0 +1,1112 @@ +import os +import tempfile +import uuid +import logging +import sys +import base64 +import json +import re +import io +import cairosvg +import pathlib +from PIL import Image, ImageDraw, ImageFont +from flask import Flask, request, jsonify, send_from_directory, send_file +from werkzeug.utils import secure_filename +from werkzeug.exceptions import RequestEntityTooLarge +from dotenv import load_dotenv +from flask_cors import CORS +from chunked_upload import chunked_upload_bp +from auth import require_auth, lenient_auth +import pdfkit +from pdfkit.configuration import Configuration +from bs4 import BeautifulSoup + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout) + ] +) +logger = logging.getLogger('video_query') + +# Load environment variables from .env file +load_dotenv() + +from video_processor import VideoProcessor + +app = Flask(__name__) +# Enable CORS with permissive settings for large file uploads +CORS(app, resources={r"/api/*": { + "origins": ["https://ai-sandbox.oliver.solutions"], + "supports_credentials": True, + "methods": ["GET", "POST", "OPTIONS"], + "allow_headers": ["Content-Type", "X-Requested-With", "Authorization"] +}}, expose_headers=["Content-Disposition", "Authorization"]) + +# Register the chunked upload blueprint +app.register_blueprint(chunked_upload_bp) + +# Configuration +UPLOAD_FOLDER = os.path.join(tempfile.gettempdir(), 'video_query_uploads') +# 5GB max upload size +MAX_CONTENT_LENGTH = 5 * 1024 * 1024 * 1024 + +# Create upload folder if it doesn't exist +os.makedirs(UPLOAD_FOLDER, exist_ok=True) + +# Configuration for persistent output - commented out as no longer needed +# PERSISTENT_PNG_ROOT_DIR = '/var/www/html/video_query/png_output' # Filesystem path for PNG files +# PERSISTENT_SVG_ROOT_DIR = '/var/www/html/video_query/svg_output' # Filesystem path for SVG files +# PERSISTENT_PNG_BASE_URL = 'https://ai-sandbox.oliver.solutions/video_query/png_output' # Web accessible URL base for PNGs +# PERSISTENT_SVG_BASE_URL = 'https://ai-sandbox.oliver.solutions/video_query/svg_output' # Web accessible URL base for SVGs + +# Create temporary directories for PDF generation instead +TEMP_PNG_DIR = os.path.join(tempfile.gettempdir(), 'video_query_png_temp') +TEMP_SVG_DIR = os.path.join(tempfile.gettempdir(), 'video_query_svg_temp') +os.makedirs(TEMP_PNG_DIR, exist_ok=True) +os.makedirs(TEMP_SVG_DIR, exist_ok=True) + +# Configure the app +app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER +app.config['MAX_CONTENT_LENGTH'] = MAX_CONTENT_LENGTH +# Set larger buffer size for large file uploads +app.config['MAX_CONTENT_PATH'] = 5 * 1024 * 1024 * 1024 # 5GB + +# Initialize video processor +video_processor = VideoProcessor() + +# Set allowed extensions for videos +ALLOWED_EXTENSIONS = {'mp4', 'avi', 'mov', 'wmv', 'mkv', 'webm'} + +def allowed_file(filename): + """Check if file has an allowed extension""" + return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + +@app.route('/api/process', methods=['POST']) +@lenient_auth +def process_video(): + """Process uploaded video with the selected mode and prompt""" + logger.info("API request received: /api/process") + logger.info(f"Content-Type: {request.content_type}") + logger.info(f"Content-Length: {request.content_length}") + + # Handle chunked upload case + if request.is_json: + data = request.get_json() + file_path = data.get('file_path') + filename = data.get('filename') + prompt = data.get('prompt') + + if not file_path or not os.path.exists(file_path): + logger.error(f"File path not found: {file_path}") + return jsonify({'success': False, 'message': 'Uploaded file not found'}), 400 + + if not prompt: + logger.error("No prompt provided") + return jsonify({'success': False, 'message': 'No prompt provided'}), 400 + + # Get user email from authentication if available + user_email = "anonymous" + if hasattr(request, "user") and isinstance(request.user, dict): + user_email = request.user.get("email", request.user.get("preferred_username", "anonymous")) + + logger.info(f"Processing chunked upload from {file_path} ({filename}) for user: {user_email}") + result = video_processor.process_video(file_path, prompt, user_email) + + # Clean up the uploaded file + try: + os.remove(file_path) + logger.info(f"Cleaned up temporary file: {file_path}") + except Exception as cleanup_error: + logger.warning(f"Could not remove temporary file {file_path}: {str(cleanup_error)}") + + if result['success']: + content_length = len(result['content']) if result['content'] else 0 + logger.info(f"Returning successful response with {content_length} characters") + return jsonify({ + 'success': True, + 'content': result['content'] + }) + else: + logger.error(f"Processing failed: {result['message']}") + return jsonify({ + 'success': False, + 'message': result['message'] + }), 500 + + # Standard direct upload method (for small files) + # Check if a file was uploaded + if 'video' not in request.files: + logger.error("No video file in request") + return jsonify({'success': False, 'message': 'No video file provided'}), 400 + + file = request.files['video'] + prompt = request.form.get('prompt', '') + + logger.info(f"Received file: {file.filename}") + logger.info(f"Prompt length: {len(prompt)} characters") + + # Check for empty filename + if file.filename == '': + logger.error("Empty filename provided") + return jsonify({'success': False, 'message': 'No video selected'}), 400 + + if not prompt: + logger.error("No prompt provided") + return jsonify({'success': False, 'message': 'No prompt provided'}), 400 + + # Check file extension + if not allowed_file(file.filename): + logger.error(f"Invalid file type: {file.filename}") + return jsonify({ + 'success': False, + 'message': f'Invalid file type. Allowed types: {", ".join(ALLOWED_EXTENSIONS)}' + }), 400 + + try: + # Make sure upload directory exists + os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) + logger.info(f"Upload directory: {app.config['UPLOAD_FOLDER']}") + + # Generate a unique filename to prevent collisions + original_filename = secure_filename(file.filename) + unique_filename = f"{uuid.uuid4()}_{original_filename}" + file_path = os.path.join(app.config['UPLOAD_FOLDER'], unique_filename) + logger.info(f"Writing to: {file_path}") + + # Stream the file to disk in larger chunks for better performance + chunk_size = 1024 * 1024 # 1MB chunks + total_bytes = 0 + try: + with open(file_path, 'wb') as f: + while True: + chunk = file.read(chunk_size) + if not chunk: + break + total_bytes += len(chunk) + f.write(chunk) + # Periodically log progress for large files + if total_bytes % (50 * 1024 * 1024) == 0: # Log every 50MB + logger.info(f"Upload progress: {total_bytes / (1024 * 1024):.2f} MB") + except Exception as chunk_error: + logger.error(f"Error during chunked upload: {str(chunk_error)}") + raise + + logger.info(f"File saved: {file_path} ({total_bytes} bytes)") + + # Get user email from authentication if available + user_email = "anonymous" + if hasattr(request, "user") and isinstance(request.user, dict): + user_email = request.user.get("email", request.user.get("preferred_username", "anonymous")) + + # Process the video + logger.info(f"Starting video processing for user: {user_email}...") + result = video_processor.process_video(file_path, prompt, user_email) + logger.info(f"Processing result: success={result['success']}") + + # Clean up the file after processing + try: + os.remove(file_path) + logger.info(f"Cleaned up temporary file: {file_path}") + except Exception as cleanup_error: + logger.warning(f"Could not remove temporary file {file_path}: {str(cleanup_error)}") + + if result['success']: + content_length = len(result['content']) if result['content'] else 0 + logger.info(f"Returning successful response with {content_length} characters") + return jsonify({ + 'success': True, + 'content': result['content'] + }) + else: + logger.error(f"Processing failed: {result['message']}") + return jsonify({ + 'success': False, + 'message': result['message'] + }), 500 + + except RequestEntityTooLarge: + logger.error(f"File too large: {request.content_length} bytes") + return jsonify({ + 'success': False, + 'message': 'The uploaded file is too large (max 5GB)' + }), 413 + except Exception as e: + import traceback + error_trace = traceback.format_exc() + logger.error(f"Error processing video: {str(e)}") + logger.error(error_trace) + return jsonify({ + 'success': False, + 'message': f'An unexpected error occurred: {str(e)}' + }), 500 + +# Test route to verify authentication +@app.route('/api/auth-test', methods=['GET']) +@lenient_auth +def auth_test(): + """Test endpoint to verify authentication is working""" + user_info = { + "authenticated": True, + "user": request.user.get("name", "Anonymous") if hasattr(request, "user") else "Unknown", + "token_present": "Authorization" in request.headers, + "token_info": {k: request.user.get(k) for k in ["name", "preferred_username", "email"] + if k in request.user} if hasattr(request, "user") else {} + } + logger.info(f"Auth test: {user_info}") + return jsonify(user_info) + +# Handle PDF generation +@app.route('/api/generate-pdf', methods=['POST']) +@lenient_auth +def generate_pdf(): + """Generate a PDF from HTML content with mermaid diagrams""" + logger.info("API request received: /api/generate-pdf") + + if not request.is_json: + logger.error("Request is not JSON") + return jsonify({'success': False, 'message': 'JSON request required'}), 400 + + data = request.get_json() + html_content = data.get('html') + text_diagrams = data.get('textDiagrams', {}) + svg_diagrams = data.get('svgDiagrams', {}) + diagram_png_data_urls = data.get('diagramPngs', {}) + + # Log detailed request information + logger.info(f"Request data: HTML content length: {len(html_content) if html_content else 0}") + logger.info(f"Text diagrams received: {len(text_diagrams)}") + logger.info(f"SVG diagrams received: {len(svg_diagrams)}") + logger.info(f"Diagram PNGs received: {len(diagram_png_data_urls)}") + + # Comment out full HTML content logging + # logger.info("HTML CONTENT RECEIVED START -------------------") + # logger.info(html_content) + # logger.info("HTML CONTENT RECEIVED END ---------------------") + + if text_diagrams: + logger.info(f"Text diagram keys: {list(text_diagrams.keys())}") + + if svg_diagrams: + logger.info(f"SVG diagram keys: {list(svg_diagrams.keys())}") + for key, value in svg_diagrams.items(): + logger.info(f"SVG diagram {key}: starts with data:image/svg+xml;base64: {value.startswith('data:image/svg+xml;base64,') if value else False}") + + if diagram_png_data_urls: + logger.info(f"Diagram PNG keys: {list(diagram_png_data_urls.keys())}") + for key, value in diagram_png_data_urls.items(): + logger.info(f"Diagram PNG {key}: starts with data:image/png;base64: {value.startswith('data:image/png;base64,') if value else False} (length: {len(value) if value else 0})") + + if not html_content: + logger.error("No HTML content provided") + return jsonify({'success': False, 'message': 'No HTML content provided'}), 400 + + try: + # Create a temporary directory for PDF and HTML file, not necessarily for images + temp_dir_for_pdf = tempfile.mkdtemp() + pdf_path = os.path.join(temp_dir_for_pdf, f"response_{uuid.uuid4()}.pdf") + + # Process HTML to replace mermaid divs with image tags + processed_html = html_content + processed_svg_ids = set() + + # Decide whether to use web URLs or file URIs for pdfkit + # Always use file:/// URIs with enable-local-file-access + # USE_WEB_URLS_FOR_PDFKIT = False # This is no longer needed + # We now use temp directories and local file paths for all images + + # Create a subdirectory for images in the temp dir (for the HTML structure) + img_dir = os.path.join(temp_dir_for_pdf, "images") + os.makedirs(img_dir, exist_ok=True) + + logger.info("HTML content before processing:") + logger.info(f"HTML contains '.mermaid' class: {'class=mermaid' in html_content}") + logger.info(f"HTML contains mermaid code blocks: {'```mermaid' in html_content or 'graph TD' in html_content}") + + # First approach: Manually look for the mermaid pattern in the HTML before any processing + pattern1 = r']*class=.?mermaid.?[^>]*>(.*?)' + pattern2 = r'
(graph\s+TD.*?)
' + pattern3 = r'graph\s+TD' + + mermaid_matches1 = re.findall(pattern1, html_content, re.DOTALL) + mermaid_matches2 = re.findall(pattern2, html_content, re.DOTALL) + mermaid_matches3 = re.findall(pattern3, html_content, re.DOTALL) + + logger.info(f"Mermaid div matches: {len(mermaid_matches1)}") + if mermaid_matches1: + for i, m in enumerate(mermaid_matches1): + logger.info(f"Mermaid div content {i} (first 100 chars): {m[:100]}") + + logger.info(f"Mermaid code block matches: {len(mermaid_matches2)}") + if mermaid_matches2: + for i, m in enumerate(mermaid_matches2): + logger.info(f"Mermaid code content {i} (first 100 chars): {m[:100]}") + + logger.info(f"Mermaid graph TD matches: {len(mermaid_matches3)}") + + # First, prioritize using the frontend-generated PNGs if available + if diagram_png_data_urls: + logger.info(f"Processing {len(diagram_png_data_urls)} PNG diagrams provided by frontend.") + + # Parse the HTML with BeautifulSoup ONCE before the loop + soup = BeautifulSoup(processed_html, 'html.parser') + + for diagram_id, png_data_url in diagram_png_data_urls.items(): + unique_png_filename = f"{diagram_id}_{uuid.uuid4()}.png" + temp_png_path = os.path.join(TEMP_PNG_DIR, unique_png_filename) + + image_source_for_pdfkit = None + + try: + if not png_data_url.startswith('data:image/png;base64,'): + logger.warning(f"Unsupported PNG data URL format for {diagram_id}") + raise ValueError("Unsupported PNG data URL format") + + base64_png_content = png_data_url.split(',', 1)[1] + png_bytes = base64.b64decode(base64_png_content) + + with open(temp_png_path, 'wb') as f: + f.write(png_bytes) + + if not os.path.exists(temp_png_path) or os.path.getsize(temp_png_path) == 0: + logger.error(f"PNG for {diagram_id} (from frontend PNG) was not saved or is empty at {temp_png_path}.") + raise ValueError("PNG saving failed or empty") + + logger.info(f"Saved frontend-generated PNG for {diagram_id} to: {temp_png_path} (size: {os.path.getsize(temp_png_path)} bytes)") + + # We no longer use web URLs, always use local file path + image_source_for_pdfkit = pathlib.Path(temp_png_path).as_uri() + + alt_text = f"Diagram: {text_diagrams.get(diagram_id, diagram_id)[:50].replace('<', '<').replace('>', '>')}..." + + # --- MODIFIED REPLACEMENT using BeautifulSoup --- + target_div = soup.find('div', id=diagram_id) + if target_div: + # Create the new img tag as a BeautifulSoup object + new_img_tag_soup = soup.new_tag('img', src=image_source_for_pdfkit, alt=alt_text) + new_img_tag_soup['style'] = "max-width:100%; margin:20px auto; display:block; border:1px solid #eee;" + + # Replace the target div with our new img tag + target_div.replace_with(new_img_tag_soup) + logger.info(f"Replaced div with id='{diagram_id}' using its frontend-generated PNG (src: {image_source_for_pdfkit}) via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + else: + logger.warning(f"PNG_WARN: Could not find div with id='{diagram_id}' in the HTML to replace with frontend PNG using BeautifulSoup.") + # Fallback to replacing code block if div with ID isn't found + original_code_for_png = text_diagrams.get(diagram_id) + if original_code_for_png: + # Try to find a pre/code block with matching content + code_blocks = soup.find_all('pre') + for code_block in code_blocks: + code_el = code_block.find('code') + if code_el and original_code_for_png.strip() in code_el.text.strip(): + # Create new img tag + new_img_tag_soup_fallback = soup.new_tag('img', src=image_source_for_pdfkit, alt=alt_text) + new_img_tag_soup_fallback['style'] = "max-width:100%; margin:20px auto; display:block; border:1px solid #eee;" + + # Replace the code block with the img tag + code_block.replace_with(new_img_tag_soup_fallback) + logger.info(f"PNG_WARN_RECOVERY: Replaced a code block matching content of diagram {diagram_id} with its frontend-PNG img tag via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + break + else: + logger.warning(f"PNG_WARN_FAIL: Also failed to find a code block for diagram {diagram_id} content for frontend-PNG replacement with BeautifulSoup.") + + except Exception as e_png_proc: + logger.error(f"Error processing provided PNG for diagram_id '{diagram_id}': {str(e_png_proc)}") + # Create a placeholder image indicating the error for this specific diagram + try: + img_err = Image.new('RGB', (500, 150), color=(255, 230, 230)) # Light red + draw_err = ImageDraw.Draw(img_err) + # Consider ImageFont.truetype for specific fonts/sizes if default is too small + title_font = ImageFont.load_default() + text_font = ImageFont.load_default() + draw_err.text((10, 10), f"Error rendering diagram:", fill=(128, 0, 0), font=title_font) + draw_err.text((10, 30), f"ID: {diagram_id}", fill=(100, 0, 0), font=text_font) + draw_err.text((10, 50), f"Details: {str(e_png_proc)[:80]}", fill=(100, 0, 0), font=text_font) + if text_diagrams.get(diagram_id): + draw_err.text((10,70), f"Code: {text_diagrams[diagram_id][:60]}...", fill=(100,0,0), font=text_font) + + with open(temp_png_path, 'wb') as f_err: # Save error image with the same name pattern + img_err.save(f_err, 'PNG') + logger.info(f"Created error placeholder image for {diagram_id} at {temp_png_path}") + + # We no longer use web URLs, always use local file path + image_source_for_pdfkit = pathlib.Path(temp_png_path).as_uri() + + # Find and replace the target div with the error image + target_div_err = soup.find('div', id=diagram_id) + if target_div_err: + new_err_img_tag = soup.new_tag('img', src=image_source_for_pdfkit, alt=f"Error rendering diagram {diagram_id}") + new_err_img_tag['style'] = "max-width:100%; margin:20px auto; display:block; border: 2px solid red;" + target_div_err.replace_with(new_err_img_tag) + logger.info(f"Replaced div with id='{diagram_id}' using an error placeholder image via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + else: + logger.error(f"Could not find div with id='{diagram_id}' to replace with error placeholder image.") + except Exception as e_placeholder_img: + logger.error(f"Failed to create error placeholder image for {diagram_id}: {str(e_placeholder_img)}") + # Try to insert a simple error paragraph if div is found + target_div_err2 = soup.find('div', id=diagram_id) + if target_div_err2: + error_p = soup.new_tag('p') + error_p['style'] = "color:red; border:1px solid red; padding:10px;" + error_p.string = f"[Error processing diagram: {diagram_id} - {str(e_png_proc)[:50]}]" + target_div_err2.replace_with(error_p) + logger.info(f"Replaced div with id='{diagram_id}' with a simple error message via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + + # After processing all PNG diagrams, update processed_html + processed_html = str(soup) + logger.info("Completed BeautifulSoup processing of all PNG diagrams") + + # Fallback to using SVG diagrams if provided + if svg_diagrams: + logger.info(f"Processing {len(svg_diagrams)} SVG diagrams provided by frontend.") + + # Ensure we're working with a BeautifulSoup object + if 'soup' not in locals() or not isinstance(soup, BeautifulSoup): + soup = BeautifulSoup(processed_html, 'html.parser') + + for diagram_id, svg_data_url in svg_diagrams.items(): + # Skip if this diagram ID was already processed in the PNG section + if diagram_id in processed_svg_ids: + logger.info(f"Skipping SVG for diagram_id '{diagram_id}' as it was already processed in PNG section.") + continue + + # Generate a unique filename for the persistent storage to avoid collisions + unique_png_filename = f"{diagram_id}_{uuid.uuid4()}.png" + temp_png_path = os.path.join(TEMP_PNG_DIR, unique_png_filename) + + image_source_for_pdfkit = None + + try: + logger.info(f"Processing diagram ID: {diagram_id}") + + if not svg_data_url.startswith('data:image/svg+xml;base64,'): + logger.warning(f"Unsupported SVG data URL format for {diagram_id}: {svg_data_url[:30]}...") + raise ValueError("Unsupported SVG data URL format") + + # Extract base64 content + base64_data = svg_data_url.split(',', 1)[1] + logger.info(f"Base64 data length: {len(base64_data)}") + + # Decode the base64 data + svg_bytes = base64.b64decode(base64_data) + logger.info(f"Decoded SVG data length: {len(svg_bytes)}") + + # Save the SVG data to the temporary SVG directory + temp_svg_filename = f"{diagram_id}_{uuid.uuid4()}.svg" + temp_svg_path = os.path.join(TEMP_SVG_DIR, temp_svg_filename) + with open(temp_svg_path, 'wb') as f: + f.write(svg_bytes) + logger.info(f"Saved SVG data to {temp_svg_path} (size: {len(svg_bytes)} bytes)") + + # Convert SVG to PNG using cairosvg with white background + png_data = cairosvg.svg2png(bytestring=svg_bytes, scale=2.0, background_color="white") + with open(temp_png_path, 'wb') as f: + f.write(png_data) + + if not os.path.exists(temp_png_path) or os.path.getsize(temp_png_path) == 0: + logger.error(f"PNG for {diagram_id} (from SVG) was not created or is empty at {temp_png_path}.") + raise ValueError("PNG creation failed or empty") + + logger.info(f"Generated PNG for {diagram_id} from SVG: {temp_png_path} (size: {os.path.getsize(temp_png_path)} bytes)") + + # We no longer use web URLs, always use local file path + image_source_for_pdfkit = pathlib.Path(temp_png_path).as_uri() + + alt_text = f"Mermaid Diagram: {text_diagrams.get(diagram_id, diagram_id)[:50].replace('<', '<').replace('>', '>')}..." + + # --- MODIFIED REPLACEMENT using BeautifulSoup --- + target_div_svg = soup.find('div', id=diagram_id) + if target_div_svg: + # Create the new img tag as a BeautifulSoup object + new_img_tag_soup_svg = soup.new_tag('img', src=image_source_for_pdfkit, alt=alt_text) + new_img_tag_soup_svg['style'] = "max-width:100%; margin:20px auto; display:block; border:1px solid #eee;" + + # Replace the target div with our new img tag + target_div_svg.replace_with(new_img_tag_soup_svg) + logger.info(f"Replaced div with id='{diagram_id}' using its SVG-generated PNG via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + else: + logger.warning(f"SVG_WARN: Could not find div with id='{diagram_id}' for SVG replacement using BeautifulSoup.") + # Try to find a code block with matching content from textDiagrams + original_code_for_svg = text_diagrams.get(diagram_id) + if original_code_for_svg and os.path.exists(persistent_png_path): + # Try to find matching code blocks + code_blocks = soup.find_all('pre') + for code_block in code_blocks: + code_el = code_block.find('code') + if code_el and original_code_for_svg.strip() in code_el.text.strip(): + # Create new img tag + new_img_tag_soup_svg_fallback = soup.new_tag('img', src=image_source_for_pdfkit, alt=alt_text) + new_img_tag_soup_svg_fallback['style'] = "max-width:100%; margin:20px auto; display:block; border:1px solid #eee;" + + # Replace the code block with the img tag + code_block.replace_with(new_img_tag_soup_svg_fallback) + logger.info(f"SVG_WARN_RECOVERY: Replaced a code block matching content for diagram {diagram_id} with its SVG-PNG img tag via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + break + else: + logger.warning(f"SVG_WARN_FAIL: Failed to find a matching code block for SVG diagram {diagram_id} with BeautifulSoup.") + + except Exception as e_svg_proc: + logger.error(f"Error processing provided SVG for diagram_id '{diagram_id}': {str(e_svg_proc)}") + # Create a placeholder image indicating the error for this specific diagram + try: + img_err = Image.new('RGB', (500, 150), color=(255, 230, 230)) # Light red + draw_err = ImageDraw.Draw(img_err) + # Consider ImageFont.truetype for specific fonts/sizes if default is too small + title_font = ImageFont.load_default() + text_font = ImageFont.load_default() + draw_err.text((10, 10), f"Error rendering diagram:", fill=(128, 0, 0), font=title_font) + draw_err.text((10, 30), f"ID: {diagram_id}", fill=(100, 0, 0), font=text_font) + draw_err.text((10, 50), f"Details: {str(e_svg_proc)[:80]}", fill=(100, 0, 0), font=text_font) + if text_diagrams.get(diagram_id): + draw_err.text((10,70), f"Code: {text_diagrams[diagram_id][:60]}...", fill=(100,0,0), font=text_font) + + with open(temp_png_path, 'wb') as f_err: # Save error image with the same name pattern + img_err.save(f_err, 'PNG') + logger.info(f"Created error placeholder image for SVG diagram {diagram_id} at {temp_png_path}") + + # We no longer use web URLs, always use local file path + image_source_for_pdfkit = pathlib.Path(temp_png_path).as_uri() + + # Find and replace the target div with the error image + target_div_svg_err = soup.find('div', id=diagram_id) + if target_div_svg_err: + new_err_img_tag_svg = soup.new_tag('img', src=image_source_for_pdfkit, alt=f"Error rendering SVG diagram {diagram_id}") + new_err_img_tag_svg['style'] = "max-width:100%; margin:20px auto; display:block; border: 2px solid red;" + target_div_svg_err.replace_with(new_err_img_tag_svg) + logger.info(f"Replaced div with id='{diagram_id}' using an SVG error placeholder image via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + else: + logger.error(f"Could not find div with id='{diagram_id}' to replace with SVG error placeholder image.") + except Exception as e_placeholder_img: + logger.error(f"Failed to create SVG error placeholder image for {diagram_id}: {str(e_placeholder_img)}") + # Try to insert a simple error paragraph if div is found + target_div_svg_err2 = soup.find('div', id=diagram_id) + if target_div_svg_err2: + error_p_svg = soup.new_tag('p') + error_p_svg['style'] = "color:red; border:1px solid red; padding:10px;" + error_p_svg.string = f"[Error processing SVG diagram: {diagram_id} - {str(e_svg_proc)[:50]}]" + target_div_svg_err2.replace_with(error_p_svg) + logger.info(f"Replaced div with id='{diagram_id}' with a simple SVG error message via BeautifulSoup.") + processed_svg_ids.add(diagram_id) + + # After processing all SVG diagrams, update processed_html + processed_html = str(soup) + logger.info("Completed BeautifulSoup processing of all SVG diagrams") + + # Fallback for any mermaid code blocks/divs *not* covered by processed_svg_ids + # This typically means the frontend didn't send an SVG for them, or all replacement attempts above failed. + + # Fallback for remaining
(that might not have had a corresponding SVG) + logger.info("Fallback: Looking for any remaining
not already handled.") + temp_processed_html_list = [] + last_end = 0 + # More specific regex for class="mermaid" and also capturing ID if present + div_fallback_pattern = r'(]*class\s*=\s*["\']?[^"\']*mermaid[^"\']*["\']?[^>]*>(?:.*?)
)' + + for match_obj in re.finditer(div_fallback_pattern, processed_html, flags=re.DOTALL | re.IGNORECASE): + start, end = match_obj.span() + div_html_segment = match_obj.group(1) + + # Check if this div has an ID that was already processed + id_in_div_match = re.search(r'\bid\s*=\s*["\']?([^"\s\'<>]+)["\']?', div_html_segment, re.IGNORECASE) + current_div_id = None + if id_in_div_match: + current_div_id = id_in_div_match.group(1) + if current_div_id in processed_svg_ids: + # Skip this div as it was already processed by svgDiagrams + temp_processed_html_list.append(processed_html[last_end:end]) + last_end = end + logger.info(f"Fallback Div: Skipping div with id='{current_div_id}' as it's in processed_svg_ids.") + continue + + # This div was not handled by a provided SVG. Generate text-based placeholder. + logger.warning(f"Fallback Div: Processing
at {start}-{end} (ID: {current_div_id}) not in processed_svg_ids. Generating text placeholder.") + + # Try to extract diagram code from the div + soup_div = BeautifulSoup(div_html_segment, 'html.parser') + diagram_text_content = soup_div.get_text(separator='\n', strip=True) or "No text in div" + + # Also check if we have this in textDiagrams + if current_div_id and current_div_id in text_diagrams: + diagram_text_content = text_diagrams[current_div_id] + + # Generate a unique ID for this fallback image + fallback_uuid = str(uuid.uuid4())[:8] + placeholder_img_name = f"fallback_div_{fallback_uuid}.png" + placeholder_path = os.path.join(TEMP_PNG_DIR, placeholder_img_name) + + try: + img = Image.new('RGB', (800, 300), color=(240, 240, 240)) + draw = ImageDraw.Draw(img) + draw.text((10, 10), "Mermaid Diagram (Fallback Render)", fill=(50, 50, 50)) + draw.text((10, 30), f"ID: {current_div_id or 'N/A'}", fill=(50, 50, 50)) + y_pos = 50 + for i, line in enumerate(diagram_text_content.split('\n')[:15]): # Show more lines + draw.text((10, y_pos), line[:80], fill=(50, 50, 50)) + y_pos += 15 + with open(placeholder_path, 'wb') as f: + img.save(f, 'PNG') + + # We no longer use web URLs, always use local file path + fallback_image_src = pathlib.Path(placeholder_path).as_uri() + img_tag = f'Fallback Mermaid Diagram' + except Exception as e_pil: + logger.error(f"Fallback Div: Error creating image for {current_div_id}: {e_pil}") + img_tag = f"
[Mermaid Diagram Code]:\n{diagram_text_content[:500]}
" + + temp_processed_html_list.append(processed_html[last_end:start]) + temp_processed_html_list.append(img_tag) + last_end = end + if current_div_id: + processed_svg_ids.add(current_div_id) # Mark as handled + + # Add the remaining content after the last match + temp_processed_html_list.append(processed_html[last_end:]) + processed_html = "".join(temp_processed_html_list) + + # Process any remaining mermaid code blocks that weren't already handled + logger.info("Fallback: Looking for any remaining mermaid code blocks not explicitly handled by ID.") + # More specific pattern for 
... or similar structures
+        # Avoid overly broad patterns like raw 'graph TD'
+        # This pattern tries to capture the code within a language-mermaid block
+        code_block_pattern = r'(]*>\s*]*class\s*=\s*["\']?[^"\']*language-mermaid[^"\']*["\']?[^>]*>([\s\S]*?)\s*
)' + + temp_processed_html_list_codeblocks = [] + last_end_codeblocks = 0 + + for match_obj in re.finditer(code_block_pattern, processed_html, flags=re.DOTALL | re.IGNORECASE): + start, end = match_obj.span() + full_match_html = match_obj.group(1) # The whole 
...
+ diagram_content = match_obj.group(2).strip() # Just the code + + logger.info(f"Found potential unhandled mermaid code block. Content starts: {diagram_content[:50]}...") + + # Try to find if this diagram_content matches any ID in textDiagrams + # And if that ID has *already* been processed (i.e., an tag was made) + is_already_processed_by_id = False + matched_original_id = None + for diag_id, original_code_from_textdiagrams in text_diagrams.items(): + # Simple check: if the extracted diagram_content is very similar to original_code + # This might need a more sophisticated similarity check. + if diagram_content == original_code_from_textdiagrams.strip(): + matched_original_id = diag_id + if diag_id in processed_svg_ids: + is_already_processed_by_id = True + logger.info(f"Code block content matches diagram ID '{diag_id}' which is in processed_svg_ids. Skipping fallback.") + break + else: + logger.info(f"Code block content matches diagram ID '{diag_id}' which was NOT in processed_svg_ids. Will attempt SVG render if available.") + break # Found a match, even if not processed by ID yet + + temp_processed_html_list_codeblocks.append(processed_html[last_end_codeblocks:start]) # Content before this match + + if is_already_processed_by_id: + # This code block corresponds to an image already inserted. + # The original 
 block should be removed or replaced by the image if it wasn't already.
+                # Since the image replacement by ID targets 
, this 
 might still be there.
+                # For safety, if it was already processed, we should ensure this 
 block is GONE.
+                # However, the primary image replacement should have taken care of the visual aspect.
+                # If the pre block is still there, it's a problem with the primary replacement not being thorough.
+                # For now, let's assume if is_already_processed_by_id, we don't want to add anything new here.
+                # We might actually want to ensure this 'full_match_html' is *removed* if its corresponding img is present.
+                # This gets complex. Let's first focus on not *adding* duplicates.
+                # If an image was already made, we effectively want to remove this 
 block.
+                # So, we append nothing here for this specific match.
+                logger.info(f"Skipping rendering for code block of diagram {matched_original_id} as it was already processed by ID.")
+                # Effectively, this removes the 
 block if its content was for an already-rendered image.
+            else:
+                # This code block was NOT processed by ID (or didn't match any known ID).
+                # Try to render it now.
+                img_tag_for_code_block = None
+                # Check if we have an SVG for it (if matched_original_id was found but not in processed_svg_ids)
+                if matched_original_id and matched_original_id in svg_diagrams and svg_diagrams[matched_original_id].startswith('data:image/svg+xml;base64,'):
+                    # Generate PNG from SVG
+                    try:
+                        base64_data = svg_diagrams[matched_original_id].split(',')[1]
+                        svg_data_decoded = base64.b64decode(base64_data)
+                        uuid_value = uuid.uuid4()
+                        
+                        # Save SVG data
+                        temp_svg_filename = f"{matched_original_id}_{uuid_value}.svg"
+                        temp_svg_path = os.path.join(TEMP_SVG_DIR, temp_svg_filename)
+                        with open(temp_svg_path, 'wb') as f:
+                            f.write(svg_data_decoded)
+                        logger.info(f"Saved SVG data for code block to {temp_svg_path} (size: {len(svg_data_decoded)} bytes)")
+                        
+                        # Save PNG data
+                        temp_png_filename = f"{matched_original_id}_{uuid_value}.png"
+                        temp_png_path = os.path.join(TEMP_PNG_DIR, temp_png_filename)
+                        
+                        png_data = cairosvg.svg2png(bytestring=svg_data_decoded, scale=2.0, background_color="white")
+                        with open(temp_png_path, 'wb') as f:
+                            f.write(png_data)
+                        
+                        # We no longer use web URLs, always use local file path
+                        img_src = pathlib.Path(temp_png_path).as_uri()
+                            
+                        img_tag_for_code_block = f'Mermaid Diagram'
+                        logger.info(f"Used SVG render for code block: {matched_original_id}")
+                        processed_svg_ids.add(matched_original_id)
+                    except Exception as e:
+                        logger.error(f"Error converting SVG to PNG: {str(e)}")
+
+                if not img_tag_for_code_block: # No SVG or SVG processing failed
+                    logger.info(f"No specific SVG found for this code block, creating PIL fallback image or pre.")
+                    # Create a fallback image if no matching SVG was found
+                    fallback_uuid_code = str(uuid.uuid4())[:8]
+                    placeholder_img_name_code = f"code_block_pil_{fallback_uuid_code}.png"
+                    placeholder_path_code = os.path.join(TEMP_PNG_DIR, placeholder_img_name_code)
+                    
+                    try:
+                        # Create an image with the diagram code
+                        img = Image.new('RGB', (800, 400), color=(245, 245, 245))
+                        draw = ImageDraw.Draw(img)
+                        draw.text((10, 10), "Mermaid Diagram (Fallback)", fill=(50, 50, 50))
+                        
+                        # Add the diagram code content
+                        y_pos = 40
+                        for line_idx, line in enumerate(diagram_content.split('\n')[:20]):
+                            draw.text((10, y_pos), line[:80], fill=(50, 50, 50))
+                            y_pos += 15
+                        
+                        with open(placeholder_path_code, 'wb') as f:
+                            img.save(f, 'PNG')
+                        
+                        # We no longer use web URLs, always use local file path
+                        img_src_code = pathlib.Path(placeholder_path_code).as_uri()
+                            
+                        img_tag_for_code_block = f'Mermaid Diagram (Code Fallback)'
+                    except Exception as e_img:
+                        logger.error(f"Error creating fallback image: {str(e_img)}")
+                        # IMPORTANT: Avoid just dumping the diagram_content here if that's the source of the problem.
+                        # Use a more generic placeholder instead
+                        img_tag_for_code_block = f'
[Mermaid diagram code could not be rendered here. Content: {diagram_content[:80]}...]
'
+
+                temp_processed_html_list_codeblocks.append(img_tag_for_code_block or "") # Append the new image/placeholder
+
+            last_end_codeblocks = end
+        
+        temp_processed_html_list_codeblocks.append(processed_html[last_end_codeblocks:])
+        processed_html = "".join(temp_processed_html_list_codeblocks)
+        
+        # Configure PDF options
+        options = {
+            'page-size': 'Letter',
+            'margin-top': '0.75in',
+            'margin-right': '0.75in',
+            'margin-bottom': '0.75in',
+            'margin-left': '0.75in',
+            'encoding': 'UTF-8',
+            # 'no-outline': None, # Removed - not supported in unpatched Qt
+            'enable-local-file-access': True  # Still needed for local file access
+            # 'load-error-handling': 'skip', # or 'ignore' - might hide issues but prevent PDF failure
+            # 'load-media-error-handling': 'skip',
+        }
+        
+        # The server has an unpatched version of wkhtmltopdf which doesn't support
+        # the 'enable-remote-images' option. We're using file:/// URIs with enable-local-file-access instead
+        
+        # Add custom CSS for better formatting
+        css = """
+        body { 
+            font-family: Arial, sans-serif; 
+            font-size: 12pt; 
+            line-height: 1.6;
+        }
+        img {
+            max-width: 100%;
+            height: auto;
+            margin: 20px auto;
+            display: block;
+        }
+        h1, h2, h3, h4, h5, h6 {
+            color: #333;
+            margin-top: 20px;
+            margin-bottom: 10px;
+        }
+        pre {
+            background-color: #f5f5f5;
+            padding: 10px;
+            border-radius: 5px;
+            overflow-x: auto;
+        }
+        code {
+            font-family: 'Courier New', Courier, monospace;
+            font-size: 11pt;
+        }
+        table {
+            border-collapse: collapse;
+            width: 100%;
+            margin: 20px 0;
+        }
+        table, th, td {
+            border: 1px solid #ddd;
+        }
+        th, td {
+            padding: 8px;
+            text-align: left;
+        }
+        th {
+            background-color: #f2f2f2;
+        }
+        /* Special handling for pre containing mermaid code */
+        pre.mermaid-source {
+            display: none;
+        }
+        """
+        
+        # Comment out final HTML content logging
+        # logger.info("====================================================")
+        # logger.info("FINAL HTML CONTENT BEING SENT TO PDFKIT:")
+        # logger.info(processed_html)
+        # logger.info("====================================================")
+        
+        # Create an index.html file in the temp directory for PDF generation
+        index_html_path = os.path.join(temp_dir_for_pdf, "index.html")
+        with open(index_html_path, 'w', encoding='utf-8') as f:
+            f.write(f"""
+            
+            
+            
+                
+                Video Query Result
+                
+            
+            
+                {processed_html}
+            
+            
+            """)
+        
+        # Log the final processed HTML for debugging
+        logger.info(f"Final HTML length: {len(processed_html)}")
+        logger.info("Final HTML contains image tags: " + str(']*src\s*=\s*["\']([^"\']+)["\']'
+        img_srcs = re.findall(img_src_pattern, processed_html)
+        logger.info(f"Found {len(img_srcs)} image sources in the HTML")
+        for i, src in enumerate(img_srcs):
+            logger.info(f"Image {i+1} src: {src}")
+        
+        # Find wkhtmltopdf on the system
+        try:
+            import subprocess
+            which_result = subprocess.run(['which', 'wkhtmltopdf'], capture_output=True, text=True)
+            if which_result.returncode == 0:
+                wkhtmltopdf_which_path = which_result.stdout.strip()
+                logger.info(f"wkhtmltopdf found at: {wkhtmltopdf_which_path}")
+            else:
+                logger.warning(f"wkhtmltopdf not found in PATH: {which_result.stderr}")
+                # Try another approach with `whereis`
+                whereis_result = subprocess.run(['whereis', 'wkhtmltopdf'], capture_output=True, text=True)
+                logger.info(f"whereis wkhtmltopdf result: {whereis_result.stdout}")
+        except Exception as e:
+            logger.warning(f"Error while trying to locate wkhtmltopdf: {str(e)}")
+        
+        try:
+            # Configure pdfkit with the path to wkhtmltopdf
+            wkhtmltopdf_path = '/usr/bin/wkhtmltopdf'  # Common location on Linux servers
+            
+            # If we found the path with 'which', use that
+            if 'wkhtmltopdf_which_path' in locals() and os.path.exists(wkhtmltopdf_which_path):
+                wkhtmltopdf_path = wkhtmltopdf_which_path
+                logger.info(f"Using wkhtmltopdf path from 'which': {wkhtmltopdf_path}")
+            
+            # Check if wkhtmltopdf is available at the specified path
+            if os.path.exists(wkhtmltopdf_path):
+                logger.info(f"Using wkhtmltopdf at: {wkhtmltopdf_path}")
+                pdfkit_config = Configuration(wkhtmltopdf=wkhtmltopdf_path)
+                pdfkit.from_file(index_html_path, pdf_path, options=options, configuration=pdfkit_config)
+            else:
+                # Try alternate paths
+                alternate_paths = [
+                    '/usr/local/bin/wkhtmltopdf',
+                    '/opt/bin/wkhtmltopdf',
+                    '/snap/bin/wkhtmltopdf'
+                ]
+                
+                found_path = None
+                for path in alternate_paths:
+                    if os.path.exists(path):
+                        found_path = path
+                        break
+                
+                if found_path:
+                    logger.info(f"Using wkhtmltopdf at alternate path: {found_path}")
+                    pdfkit_config = Configuration(wkhtmltopdf=found_path)
+                    pdfkit.from_file(index_html_path, pdf_path, options=options, configuration=pdfkit_config)
+                else:
+                    # Try with default config, which may use PATH environment variable
+                    logger.warning("wkhtmltopdf not found at expected paths, trying with default configuration")
+                    pdfkit.from_file(index_html_path, pdf_path, options=options)
+            
+            logger.info(f"PDF generated successfully, file size: {os.path.getsize(pdf_path)} bytes")
+        except Exception as pdf_error:
+            logger.error(f"Error generating PDF: {str(pdf_error)}")
+            import traceback
+            logger.error(traceback.format_exc())
+            
+            # Try with direct HTML content as fallback
+            logger.info("Trying fallback PDF generation directly from HTML string")
+            try:
+                # Strip out any remaining mermaid divs or code blocks that might be causing problems
+                final_html = processed_html
+                problem_patterns = [
+                    r']*class=.?mermaid.?[^>]*>.*?
',
+                    r'
graph\s+TD.*?
',
+                    r'
.*?
',
+                    r'```mermaid\s+[\s\S]*?```',
+                    r'graph\s+TD[^;]*;'
+                ]
+                
+                logger.info("Stripping any remaining problematic elements before final fallback")
+                for pattern in problem_patterns:
+                    before_len = len(final_html)
+                    final_html = re.sub(pattern, '
[Diagram placeholder]
', final_html, flags=re.DOTALL)
+                    after_len = len(final_html)
+                    if before_len != after_len:
+                        logger.info(f"Removed pattern, length before: {before_len}, after: {after_len}")
+                
+                fallback_options = {
+                    'page-size': 'Letter',
+                    'margin-top': '0.75in',
+                    'margin-right': '0.75in',
+                    'margin-bottom': '0.75in',
+                    'margin-left': '0.75in',
+                    'encoding': 'UTF-8',
+                    'enable-local-file-access': True
+                }
+                
+                # Try to locate wkhtmltopdf for fallback method too
+                if os.path.exists(wkhtmltopdf_path):
+                    pdfkit_config = Configuration(wkhtmltopdf=wkhtmltopdf_path)
+                    pdfkit.from_string(f"""
+                    
+                    
+                    
+                        
+                        Video Query Result
+                        
+                    
+                    
+                        {final_html}
+                    
+                    
+                    """, pdf_path, options=fallback_options, configuration=pdfkit_config)
+                else:
+                    logger.warning("Using default configuration for fallback PDF generation")
+                    pdfkit.from_string(f"""
+                    
+                    
+                    
+                        
+                        Video Query Result
+                        
+                    
+                    
+                        {final_html}
+                    
+                    
+                    """, pdf_path, options=fallback_options)
+                    
+                logger.info("Fallback PDF generation succeeded")
+            except Exception as fallback_error:
+                logger.error(f"Fallback PDF generation also failed: {str(fallback_error)}")
+        
+        # Read the generated PDF file
+        if os.path.exists(pdf_path):
+            with open(pdf_path, 'rb') as file:
+                pdf_data = file.read()
+            
+            # Encode the PDF as base64
+            pdf_base64 = base64.b64encode(pdf_data).decode('utf-8')
+            logger.info(f"Encoded PDF data length: {len(pdf_base64)}")
+        else:
+            logger.error("PDF file does not exist after generation")
+            return jsonify({'success': False, 'message': 'PDF generation failed'}), 500
+        
+        # Clean up temporary PDF generation files and temp images
+        try:
+            # Clean up PDF temp directory
+            for root, dirs, files in os.walk(temp_dir_for_pdf, topdown=False):
+                for file in files:
+                    os.remove(os.path.join(root, file))
+                for dir_name in dirs: # dir is a reserved word
+                    os.rmdir(os.path.join(root, dir_name))
+            os.rmdir(temp_dir_for_pdf)
+            logger.info(f"Cleaned up temporary PDF directory: {temp_dir_for_pdf}")
+            
+            # Clean up temporary PNG and SVG files
+            for file in os.listdir(TEMP_PNG_DIR):
+                os.remove(os.path.join(TEMP_PNG_DIR, file))
+            for file in os.listdir(TEMP_SVG_DIR):
+                os.remove(os.path.join(TEMP_SVG_DIR, file))
+            logger.info("Cleaned up temporary PNG and SVG files")
+            
+        except Exception as cleanup_error:
+            logger.warning(f"Could not remove all temporary files: {str(cleanup_error)}")
+        
+        return jsonify({
+            'success': True,
+            'pdf': pdf_base64,
+            'filename': 'video_query_result.pdf'
+        })
+        
+    except Exception as e:
+        import traceback
+        error_trace = traceback.format_exc()
+        logger.error(f"Error generating PDF: {str(e)}")
+        logger.error(error_trace)
+        return jsonify({
+            'success': False,
+            'message': f'An unexpected error occurred: {str(e)}'
+        }), 500
+
+# Handle CORS preflight requests for all API routes
+@app.route('/api/', methods=['OPTIONS'])
+def handle_options(path):
+    response = jsonify({})
+    response.headers.add('Access-Control-Allow-Origin', 'https://ai-sandbox.oliver.solutions')
+    response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization,X-Requested-With')
+    response.headers.add('Access-Control-Allow-Methods', 'GET,POST,OPTIONS')
+    response.headers.add('Access-Control-Max-Age', '86400')  # 24 hours
+    response.headers.add('Access-Control-Allow-Credentials', 'true')
+    return response
+
+# No longer need to serve frontend from the backend
+# Frontend will be hosted at https://ai-sandbox.oliver.solutions/video_query
+
+if __name__ == '__main__':
+    # For development only - use Hypercorn in production
+    app.run(debug=True, port=5000)
\ No newline at end of file
diff --git a/backend/auth.py b/backend/auth.py
new file mode 100644
index 0000000..a4647fb
--- /dev/null
+++ b/backend/auth.py
@@ -0,0 +1,198 @@
+import json
+import logging
+import requests
+from functools import wraps
+# Use more specific imports to avoid potential name conflicts
+import jose.jwt as jwt
+from jose.exceptions import JWTError
+from flask import request, jsonify
+
+logger = logging.getLogger('video_query')
+
+# Azure AD B2C configuration
+TENANT_ID = 'e519c2e6-bc6d-4fdf-8d9c-923c2f002385'
+CLIENT_ID = '9079054c-9620-4757-a256-23413042f1ef'
+JWKS_URI = f'https://login.microsoftonline.com/{TENANT_ID}/discovery/v2.0/keys'
+
+# Cache for JWKS keys
+jwks_cache = None
+jwks_last_updated = None
+
+def get_jwks():
+    """Fetch the JWKS (JSON Web Key Set) from Azure AD"""
+    global jwks_cache, jwks_last_updated
+    
+    # Use cached version if available
+    if jwks_cache:
+        return jwks_cache
+    
+    try:
+        logger.info(f"Fetching JWKS from {JWKS_URI}")
+        response = requests.get(JWKS_URI)
+        response.raise_for_status()
+        jwks_cache = response.json()
+        return jwks_cache
+    except Exception as e:
+        logger.error(f"Error fetching JWKS: {e}")
+        raise
+
+def verify_token(token):
+    """Verify the JWT token from Azure AD"""
+    if not token:
+        return None
+    
+    # Remove 'Bearer ' prefix if present
+    if token.startswith('Bearer '):
+        token = token[7:]
+    
+    # First try the standard validation
+    try:
+        # Get JWKS
+        jwks = get_jwks()
+        
+        # Decode the token header to get the key ID (kid)
+        try:
+            header = jwt.get_unverified_header(token)
+        except Exception as header_error:
+            logger.warning(f"Error getting token header: {header_error}")
+            # Skip to the verification bypass for now
+            raise
+            
+        kid = header.get('kid')
+        
+        if not kid:
+            logger.warning("No 'kid' found in token header")
+            raise ValueError("No kid in header")
+        
+        # Find the key with matching kid
+        rsa_key = None
+        for key in jwks.get('keys', []):
+            if key.get('kid') == kid:
+                rsa_key = {
+                    'kty': key.get('kty'),
+                    'kid': key.get('kid'),
+                    'use': key.get('use'),
+                    'n': key.get('n'),
+                    'e': key.get('e')
+                }
+                break
+        
+        if not rsa_key:
+            logger.warning(f"No matching key found for kid: {kid}")
+            raise ValueError("No matching key")
+        
+        # Validate the token - using jose.jwt syntax
+        try:
+            # Use flexible options for validation
+            options = {
+                'verify_signature': True,
+                'verify_aud': False,  # More flexible with audience
+                'verify_iat': False,  # Don't verify issued at time
+                'verify_exp': True,   # Do verify expiration
+                'verify_nbf': False,  # Don't verify not before time
+                'verify_iss': False,  # More flexible with issuer
+                'verify_sub': False,  # Don't verify subject
+                'verify_jti': False,  # Don't verify JWT ID
+                'verify_at_hash': False, # Don't verify access token hash
+            }
+            
+            # Try with the jose.jwt module
+            payload = jwt.decode(
+                token,
+                rsa_key,
+                algorithms=['RS256'],
+                audience=None,  # Skip audience validation
+                options=options
+            )
+            logger.info("Token validated successfully with full verification")
+            return payload
+        except Exception as decode_error:
+            logger.warning(f"Error with full token validation: {decode_error}")
+            raise
+    
+    except Exception as e:
+        logger.warning(f"Standard token validation failed, trying fallback: {e}")
+        
+        # Fallback: Parse the token without verifying signature
+        try:
+            # Just decode the payload part without verification
+            # This is not ideal for security but will get things working
+            import base64
+            import json
+            
+            # Split the token into parts
+            parts = token.split('.')
+            if len(parts) != 3:
+                logger.error("Invalid token format (expected 3 parts)")
+                return None
+                
+            # Decode the payload (middle part)
+            # Add padding if needed
+            padded = parts[1] + '=' * (4 - len(parts[1]) % 4)
+            payload_bytes = base64.urlsafe_b64decode(padded)
+            payload = json.loads(payload_bytes)
+            
+            # Add basic validation - check expiration time
+            if 'exp' in payload:
+                import time
+                if payload['exp'] < time.time():
+                    logger.error("Token is expired")
+                    return None
+            
+            logger.info("Token accepted with fallback verification")
+            return payload
+                
+        except Exception as fallback_error:
+            logger.error(f"Even fallback token verification failed: {fallback_error}")
+            return None
+
+def require_auth(f):
+    """Decorator to require authentication for Flask routes"""
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        auth_header = request.headers.get('Authorization')
+        
+        if not auth_header:
+            logger.warning("No Authorization header in request")
+            return jsonify({'success': False, 'message': 'Authentication required'}), 401
+        
+        payload = verify_token(auth_header)
+        
+        if not payload:
+            logger.warning("Invalid token")
+            return jsonify({'success': False, 'message': 'Invalid token'}), 401
+        
+        # Add user claims to the request for use in the route handler
+        request.user = payload
+        logger.info(f"Request authenticated for user: {payload.get('name', 'Unknown')}")
+        
+        return f(*args, **kwargs)
+    
+    return decorated
+
+def lenient_auth(f):
+    """Decorator with lenient authentication - attempts to validate but proceeds regardless"""
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        auth_header = request.headers.get('Authorization')
+        
+        if auth_header:
+            # Try to verify the token, but don't block if it fails
+            payload = verify_token(auth_header)
+            if payload:
+                # Add user claims to the request for use in the route handler
+                request.user = payload
+                logger.info(f"Request authenticated for user: {payload.get('name', 'Unknown')}")
+            else:
+                logger.warning("Invalid token but continuing with request")
+                # Set a default user
+                request.user = {"name": "Anonymous"}
+        else:
+            logger.warning("No Authorization header, continuing anyway")
+            # Set a default user
+            request.user = {"name": "Anonymous"}
+        
+        # Continue with the request regardless of authentication
+        return f(*args, **kwargs)
+    
+    return decorated
\ No newline at end of file
diff --git a/backend/chunked_upload.py b/backend/chunked_upload.py
new file mode 100644
index 0000000..a94e877
--- /dev/null
+++ b/backend/chunked_upload.py
@@ -0,0 +1,166 @@
+import os
+import uuid
+import json
+from flask import Blueprint, request, jsonify, current_app
+from werkzeug.utils import secure_filename
+import logging
+from auth import require_auth
+
+logger = logging.getLogger('video_query')
+
+# Create blueprint for handling chunked uploads
+chunked_upload_bp = Blueprint('chunked_upload', __name__)
+
+# Track upload sessions
+active_uploads = {}
+
+@chunked_upload_bp.route('/api/init-upload', methods=['POST'])
+@require_auth
+def init_upload():
+    """Initialize a new chunked upload session"""
+    if not request.is_json:
+        return jsonify({"success": False, "message": "Request must be JSON"}), 400
+    
+    data = request.get_json()
+    filename = data.get('filename')
+    total_size = data.get('size')
+    
+    if not filename or not total_size:
+        return jsonify({"success": False, "message": "Filename and size are required"}), 400
+    
+    # Generate a unique ID for this upload
+    upload_id = str(uuid.uuid4())
+    
+    # Generate a unique filename
+    original_filename = secure_filename(filename)
+    unique_filename = f"{upload_id}_{original_filename}"
+    upload_path = os.path.join(current_app.config['UPLOAD_FOLDER'], unique_filename)
+    
+    # Create/ensure the upload folder exists
+    os.makedirs(current_app.config['UPLOAD_FOLDER'], exist_ok=True)
+    
+    # Initialize an empty file
+    with open(upload_path, 'wb') as f:
+        pass
+    
+    # Store upload info
+    active_uploads[upload_id] = {
+        'path': upload_path,
+        'original_filename': original_filename,
+        'total_size': total_size,
+        'uploaded_size': 0,
+        'complete': False
+    }
+    
+    logger.info(f"Initialized upload session {upload_id} for {filename} ({total_size} bytes)")
+    
+    return jsonify({
+        "success": True,
+        "upload_id": upload_id
+    })
+
+@chunked_upload_bp.route('/api/upload-chunk/', methods=['POST'])
+@require_auth
+def upload_chunk(upload_id):
+    """Handle a chunk of file data"""
+    if upload_id not in active_uploads:
+        return jsonify({"success": False, "message": "Invalid upload ID"}), 400
+    
+    upload = active_uploads[upload_id]
+    
+    # Check if we have the file chunk
+    if 'chunk' not in request.files:
+        return jsonify({"success": False, "message": "No chunk in request"}), 400
+    
+    chunk = request.files['chunk']
+    chunk_number = request.form.get('chunk_number', 0, type=int)
+    
+    # Update the file with this chunk
+    with open(upload['path'], 'ab') as f:
+        chunk_data = chunk.read()
+        chunk_size = len(chunk_data)
+        f.write(chunk_data)
+    
+    # Update upload state
+    upload['uploaded_size'] += chunk_size
+    progress = min(100, round((upload['uploaded_size'] / upload['total_size']) * 100))
+    
+    logger.info(f"Received chunk {chunk_number} for upload {upload_id} - Progress: {progress}%")
+    
+    # Check if upload is complete
+    if upload['uploaded_size'] >= upload['total_size']:
+        upload['complete'] = True
+        logger.info(f"Upload complete for {upload_id}")
+    
+    return jsonify({
+        "success": True,
+        "upload_id": upload_id,
+        "chunk_number": chunk_number,
+        "bytes_received": chunk_size,
+        "total_received": upload['uploaded_size'],
+        "progress": progress,
+        "complete": upload['complete']
+    })
+
+@chunked_upload_bp.route('/api/complete-upload/', methods=['POST'])
+@require_auth
+def complete_upload(upload_id):
+    """Mark an upload as complete and return the file path for processing"""
+    if upload_id not in active_uploads:
+        return jsonify({"success": False, "message": "Invalid upload ID"}), 400
+    
+    upload = active_uploads[upload_id]
+    
+    # Verify the upload is actually complete
+    if not upload['complete']:
+        # Check the file size
+        if os.path.exists(upload['path']):
+            actual_size = os.path.getsize(upload['path'])
+            if actual_size >= upload['total_size']:
+                upload['complete'] = True
+                upload['uploaded_size'] = actual_size
+                logger.info(f"Manually verified upload complete for {upload_id}")
+            else:
+                logger.warning(f"Upload not complete for {upload_id}: {actual_size}/{upload['total_size']} bytes")
+                return jsonify({
+                    "success": False, 
+                    "message": f"Upload not complete: {actual_size}/{upload['total_size']} bytes"
+                }), 400
+        else:
+            logger.error(f"Upload file not found for {upload_id}")
+            return jsonify({"success": False, "message": "Upload file not found"}), 500
+    
+    logger.info(f"Upload {upload_id} marked as complete: {upload['original_filename']}")
+    
+    return jsonify({
+        "success": True,
+        "upload_id": upload_id,
+        "file_path": upload['path'],
+        "filename": upload['original_filename'],
+        "size": upload['uploaded_size']
+    })
+
+@chunked_upload_bp.route('/api/cancel-upload/', methods=['POST'])
+@require_auth
+def cancel_upload(upload_id):
+    """Cancel an upload and delete the partial file"""
+    if upload_id not in active_uploads:
+        return jsonify({"success": False, "message": "Invalid upload ID"}), 400
+    
+    upload = active_uploads[upload_id]
+    
+    # Delete the partial file
+    if os.path.exists(upload['path']):
+        try:
+            os.remove(upload['path'])
+            logger.info(f"Deleted partial upload for {upload_id}")
+        except Exception as e:
+            logger.error(f"Error deleting partial upload for {upload_id}: {str(e)}")
+    
+    # Remove from active uploads
+    del active_uploads[upload_id]
+    
+    return jsonify({
+        "success": True,
+        "message": "Upload cancelled"
+    })
\ No newline at end of file
diff --git a/backend/fix_jose.sh b/backend/fix_jose.sh
new file mode 100755
index 0000000..b957f89
--- /dev/null
+++ b/backend/fix_jose.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Script to fix the jose module conflict
+
+# Activate virtual environment (adjust if needed)
+source ../venv/bin/activate
+
+# Show current package information
+echo "Current jose packages:"
+pip list | grep jose
+
+# Uninstall the problematic jose package (if it exists)
+echo "Uninstalling old jose package..."
+pip uninstall -y jose
+
+# Install only python-jose
+echo "Installing python-jose properly..."
+pip uninstall -y python-jose
+pip install python-jose==3.3.0
+
+# Update requirements.txt
+echo "Updating requirements.txt..."
+pip freeze > requirements.txt
+
+echo "Fix complete. Please try running the application again."
\ No newline at end of file
diff --git a/backend/install_wkhtmltopdf.sh b/backend/install_wkhtmltopdf.sh
new file mode 100755
index 0000000..c8f543c
--- /dev/null
+++ b/backend/install_wkhtmltopdf.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+# Script to install wkhtmltopdf on the server
+
+echo "Checking for wkhtmltopdf..."
+if command -v wkhtmltopdf &> /dev/null; then
+    echo "wkhtmltopdf is already installed at $(which wkhtmltopdf)"
+    wkhtmltopdf --version
+    exit 0
+fi
+
+echo "wkhtmltopdf not found, installing..."
+
+# Detect OS
+if [ -f /etc/os-release ]; then
+    . /etc/os-release
+    OS=$ID
+    VERSION=$VERSION_ID
+    echo "Detected OS: $OS $VERSION"
+else
+    echo "Unable to detect OS, assuming Ubuntu/Debian"
+    OS="ubuntu"
+fi
+
+# Install based on OS
+if [[ "$OS" == "ubuntu" || "$OS" == "debian" ]]; then
+    echo "Installing on Ubuntu/Debian..."
+    sudo apt-get update
+    sudo apt-get install -y wkhtmltopdf
+elif [[ "$OS" == "centos" || "$OS" == "rhel" || "$OS" == "fedora" ]]; then
+    echo "Installing on CentOS/RHEL/Fedora..."
+    sudo yum install -y wkhtmltopdf
+elif [[ "$OS" == "alpine" ]]; then
+    echo "Installing on Alpine..."
+    apk add --no-cache wkhtmltopdf
+else
+    echo "Unsupported OS: $OS"
+    echo "Please install wkhtmltopdf manually."
+    exit 1
+fi
+
+# Verify installation
+if command -v wkhtmltopdf &> /dev/null; then
+    echo "wkhtmltopdf installed successfully at $(which wkhtmltopdf)"
+    wkhtmltopdf --version
+    exit 0
+else
+    echo "wkhtmltopdf installation failed."
+    echo "You may need to install it manually from https://wkhtmltopdf.org/downloads.html"
+    exit 1
+fi
\ No newline at end of file
diff --git a/backend/requirements-py310.txt b/backend/requirements-py310.txt
new file mode 100644
index 0000000..37401f1
--- /dev/null
+++ b/backend/requirements-py310.txt
@@ -0,0 +1,10 @@
+flask==2.2.5
+flask-cors==4.0.0
+hypercorn==0.14.4
+python-dotenv==1.0.0
+google-generativeai==0.3.1
+requests==2.31.0
+werkzeug==2.2.3
+python-jose[cryptography]==3.3.0
+setuptools==68.2.2
+wheel==0.41.3
\ No newline at end of file
diff --git a/backend/requirements.txt b/backend/requirements.txt
new file mode 100644
index 0000000..7e3d68b
--- /dev/null
+++ b/backend/requirements.txt
@@ -0,0 +1,55 @@
+annotated-types==0.7.0
+blinker==1.9.0
+cachetools==5.5.2
+cairocffi==1.7.1
+cairosvg==2.8.0
+certifi==2025.4.26
+cffi==1.17.1
+charset-normalizer==3.4.2
+click==8.2.0
+cssselect2==0.8.0
+defusedxml==0.7.1
+Flask==3.1.0
+flask-cors==5.0.1
+google-ai-generativelanguage==0.6.15
+google-api-core==2.25.0rc0
+google-api-python-client==2.169.0
+google-auth==2.40.0
+google-auth-httplib2==0.2.0
+google-generativeai==0.8.5
+googleapis-common-protos==1.70.0
+grpcio==1.71.0
+grpcio-status==1.71.0
+h11==0.16.0
+h2==4.2.0
+hpack==4.1.0
+httplib2==0.22.0
+Hypercorn==0.17.3
+hyperframe==6.1.0
+idna==3.10
+itsdangerous==2.2.0
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+pdfkit==1.0.0
+Pillow==11.2.1
+priority==2.0.0
+proto-plus==1.26.1
+protobuf==5.29.4
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.22
+pydantic==2.11.4
+pydantic_core==2.33.2
+pyparsing==3.2.3
+python-dotenv==1.1.0
+requests==2.32.3
+rsa==4.9.1
+tinycss2==1.4.0
+tqdm==4.67.1
+typing-inspection==0.4.0
+typing_extensions==4.13.2
+uritemplate==4.1.1
+urllib3==2.4.0
+webencodings==0.5.1
+Werkzeug==3.1.3
+wsproto==1.2.0
diff --git a/backend/run.py b/backend/run.py
new file mode 100755
index 0000000..999a671
--- /dev/null
+++ b/backend/run.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+import os
+import argparse
+from hypercorn.config import Config
+from hypercorn.asyncio import serve
+import asyncio
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+from app import app
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the Video Query backend server")
+    parser.add_argument("--host", default="0.0.0.0", help="Host IP address to bind to")
+    parser.add_argument("--port", type=int, default=5010, help="Port to bind to")
+    args = parser.parse_args()
+    
+    config = Config()
+    config.bind = [f"{args.host}:{args.port}"]
+    
+    # Set large file upload size limit (5GB)
+    config.h11_max_incomplete_size = 5 * 1024 * 1024 * 1024
+    
+    # Increase timeouts for large uploads
+    config.keep_alive_timeout = 3600  # 60 minutes
+    
+    # Set configuration values to help with large file uploads
+    config.websocket_ping_interval = 20  # Send ping frames every 20 seconds to keep connection alive
+    config.graceful_timeout = 3600  # Allow up to 1 hour for requests to complete during shutdown
+    
+    # Other helpful settings for large files
+    config.worker_class = "asyncio"
+    config.backlog = 100
+    config.read_timeout = 3600  # 60 minutes
+    config.write_timeout = 3600  # 60 minutes
+    
+    # Run with Hypercorn
+    asyncio.run(serve(app, config))
\ No newline at end of file
diff --git a/backend/test_api.py b/backend/test_api.py
new file mode 100644
index 0000000..c24be4e
--- /dev/null
+++ b/backend/test_api.py
@@ -0,0 +1,23 @@
+import google.generativeai as genai
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+print(f"API Key set: {bool(os.getenv('GOOGLE_API_KEY'))}")
+
+try:
+    # Configure client with API key
+    api_key = os.getenv("GOOGLE_API_KEY")
+    genai.configure(api_key=api_key)
+    
+    # Test connection
+    model = genai.GenerativeModel('gemini-1.5-pro')
+    response = model.generate_content('Test the API connection')
+    
+    print("API connection test successful!")
+    print(f"Response: {response.text}")
+    
+except Exception as e:
+    print(f"API error: {e}")
\ No newline at end of file
diff --git a/backend/test_webhook.py b/backend/test_webhook.py
new file mode 100644
index 0000000..a403752
--- /dev/null
+++ b/backend/test_webhook.py
@@ -0,0 +1,137 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import json
+import os
+import tempfile
+import datetime
+from video_processor import VideoProcessor
+
+class TestWebhookIntegration(unittest.TestCase):
+    """Test cases for webhook integration in VideoProcessor."""
+
+    def setUp(self):
+        """Set up test environment."""
+        # Create a VideoProcessor instance with a mock API key
+        self.video_processor = VideoProcessor(api_key="test_api_key")
+        
+        # Create a temporary file to simulate a video
+        self.temp_file = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False)
+        self.temp_file.close()
+        
+        # Write some dummy data to the file
+        with open(self.temp_file.name, 'wb') as f:
+            f.write(b'test video content')
+
+    def tearDown(self):
+        """Clean up after tests."""
+        # Remove the temporary file
+        if os.path.exists(self.temp_file.name):
+            os.unlink(self.temp_file.name)
+
+    @patch('video_processor.genai')
+    @patch('video_processor.requests.post')
+    def test_webhook_called_on_successful_processing(self, mock_post, mock_genai):
+        """Test that the webhook is called when video processing is successful."""
+        # Mock the genai API responses
+        mock_file = MagicMock()
+        mock_file.uri = "test_uri"
+        mock_file.name = "test_name"
+        mock_file.state.name = "ACTIVE"
+        mock_genai.upload_file.return_value = mock_file
+        
+        # Mock the generate_content response
+        mock_response = MagicMock()
+        mock_part = MagicMock()
+        mock_part.text = "Test response content"
+        mock_response.parts = [mock_part]
+        mock_genai.GenerativeModel.return_value.generate_content.return_value = mock_response
+        
+        # Set up the mock for the requests.post call
+        mock_post.return_value.status_code = 200
+        
+        # Test data
+        test_prompt = "Test prompt for video processing"
+        test_email = "test.user@example.com"
+        
+        # Call the process_video method
+        result = self.video_processor.process_video(
+            self.temp_file.name, 
+            test_prompt,
+            test_email
+        )
+        
+        # Verify the result is successful
+        self.assertTrue(result["success"])
+        self.assertEqual(result["content"], "Test response content")
+        
+        # Verify webhook was called with correct data
+        mock_post.assert_called_once()
+        
+        # Get the arguments the mock was called with
+        call_args = mock_post.call_args
+        
+        # Verify URL
+        self.assertEqual(call_args[0][0], "https://hook.us1.make.celonis.com/8ri1h8b2he4wudp2jku69mgcxumzxf3v")
+        
+        # Verify headers
+        self.assertEqual(call_args[1]["headers"], {"Content-Type": "application/json"})
+        
+        # Verify timeout
+        self.assertEqual(call_args[1]["timeout"], 10)
+        
+        # Parse and verify the payload
+        payload = json.loads(call_args[1]["data"])
+        self.assertEqual(payload["tool"], "VIDEOQUERY")
+        self.assertEqual(payload["user"], test_email)
+        self.assertEqual(payload["model"], "GEMINI")
+        self.assertEqual(payload["prompt"], test_prompt)
+        
+        # Verify date format (should be ISO format)
+        try:
+            datetime.datetime.fromisoformat(payload["date"])
+            date_valid = True
+        except ValueError:
+            date_valid = False
+        self.assertTrue(date_valid, "Date should be in ISO format")
+
+    @patch('video_processor.genai')
+    @patch('video_processor.requests.post')
+    def test_webhook_error_does_not_affect_processing(self, mock_post, mock_genai):
+        """Test that errors in the webhook don't affect the main processing flow."""
+        # Mock the genai API responses
+        mock_file = MagicMock()
+        mock_file.uri = "test_uri"
+        mock_file.name = "test_name"
+        mock_file.state.name = "ACTIVE"
+        mock_genai.upload_file.return_value = mock_file
+        
+        # Mock the generate_content response
+        mock_response = MagicMock()
+        mock_part = MagicMock()
+        mock_part.text = "Test response content"
+        mock_response.parts = [mock_part]
+        mock_genai.GenerativeModel.return_value.generate_content.return_value = mock_response
+        
+        # Set up the mock for the requests.post call to raise an exception
+        mock_post.side_effect = Exception("Webhook connection error")
+        
+        # Test data
+        test_prompt = "Test prompt for video processing"
+        test_email = "test.user@example.com"
+        
+        # Call the process_video method
+        result = self.video_processor.process_video(
+            self.temp_file.name, 
+            test_prompt,
+            test_email
+        )
+        
+        # Verify the result is still successful despite webhook error
+        self.assertTrue(result["success"])
+        self.assertEqual(result["content"], "Test response content")
+        
+        # Verify webhook was called
+        mock_post.assert_called_once()
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/backend/test_webhook_manual.py b/backend/test_webhook_manual.py
new file mode 100644
index 0000000..c7c69e8
--- /dev/null
+++ b/backend/test_webhook_manual.py
@@ -0,0 +1,46 @@
+"""
+Manual test script for the webhook integration.
+This script simulates a webhook call without processing a video,
+allowing us to verify the webhook is working correctly.
+"""
+
+import logging
+import sys
+from video_processor import VideoProcessor
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger('webhook_test')
+
+def test_webhook_manually():
+    """Test the webhook call manually"""
+    # Create a VideoProcessor instance
+    try:
+        processor = VideoProcessor()
+        logger.info("VideoProcessor initialized")
+        
+        # Test user email
+        test_email = "test.user@example.com"
+        
+        # Test prompt
+        test_prompt = "Test prompt for webhook verification"
+        
+        # Call the webhook method directly
+        logger.info(f"Sending test webhook call for user {test_email}")
+        processor.send_usage_webhook(test_email, test_prompt)
+        
+        logger.info("Webhook test completed")
+        
+    except Exception as e:
+        logger.error(f"Error in webhook test: {str(e)}")
+        import traceback
+        logger.error(traceback.format_exc())
+
+if __name__ == "__main__":
+    test_webhook_manually()
\ No newline at end of file
diff --git a/backend/video-query.service b/backend/video-query.service
new file mode 100644
index 0000000..31f452b
--- /dev/null
+++ b/backend/video-query.service
@@ -0,0 +1,17 @@
+[Unit]
+Description=Video Query Service
+After=network.target
+
+[Service]
+User=www-data
+WorkingDirectory=/path/to/video_query/backend
+ExecStart=/path/to/video_query/venv/bin/python run.py --port 5010
+Restart=on-failure
+# Extend timeouts for processing large videos
+TimeoutStartSec=600
+TimeoutStopSec=60
+# EnvironmentFile directive tells systemd to read environment variables from the .env file
+EnvironmentFile=/path/to/video_query/backend/.env
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/backend/video_processor.py b/backend/video_processor.py
new file mode 100644
index 0000000..b1653c0
--- /dev/null
+++ b/backend/video_processor.py
@@ -0,0 +1,224 @@
+import google.generativeai as genai
+import mimetypes
+import time
+import os
+import logging
+import requests
+import json
+import datetime
+from typing import Dict, Any, Optional
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+logger = logging.getLogger('video_query')
+
+class VideoProcessor:
+    """
+    Class to handle video uploads and processing with Gemini API.
+    """
+    # Default prompts for different modes
+    PROMPTS = {
+        "meeting_summary": "Generate a detailed summary of the meeting in the attached video recording, including discussion points and action items with owners",
+        "process_documentation": "Generate detailed process documentation suitable for reference or training based on the process illustrated in the attached video recording. Write the documentation so that a new user will be able to follow step by step and accomplish the task illustrated in the video",
+        "documentation_with_charts": "Analyze this video to create comprehensive process documentation with workflow diagrams for a knowledge base article. Follow these requirements exactly:\n\n1. CONTENT REQUIREMENTS:\n   - Provide a detailed step-by-step explanation of the process shown\n   - Be extremely verbose and thorough - include all relevant details, context, and nuances\n   - Structure as a complete knowledge base article with clear sections\n   - Include overview, detailed steps, tips, and troubleshooting where applicable\n\n2. MERMAID DIAGRAM REQUIREMENTS:\n   - Create workflow diagrams using valid Mermaid syntax where helpful\n   - CRITICAL: Use only simple alphanumeric text in node descriptions and labels\n   - CRITICAL: No special characters like quotes brackets colons semicolons or symbols in node text\n   - CRITICAL: Use underscores instead of spaces in node IDs and labels\n   - CRITICAL: Keep all text simple to avoid syntax errors\n   - Example format: Start_Process --> Complete_Task --> End_Process\n   - Use flowchart format: graph TD or graph LR\n\n3. OUTPUT STRUCTURE:\n   - Title and overview section\n   - Prerequisites section if applicable\n   - Detailed step-by-step process\n   - Mermaid workflow diagram(s) showing the process flow\n   - Tips and best practices\n   - Troubleshooting common issues\n\nEnsure all Mermaid diagrams use simple text without special characters to prevent parsing errors.",
+        "custom": ""  # Custom prompt will be provided by the user
+    }
+
+    # Maximum video duration in minutes (Gemini limitation)
+    MAX_VIDEO_DURATION = 55
+    
+    # Threshold for chunked upload (10MB)
+    CHUNKED_UPLOAD_THRESHOLD = 10 * 1024 * 1024
+    
+    # Webhook URL for tracking usage
+    WEBHOOK_URL = "https://hook.us1.make.celonis.com/8ri1h8b2he4wudp2jku69mgcxumzxf3v"
+
+    def __init__(self, api_key: Optional[str] = None):
+        """Initialize with API key from environment variable or direct setting"""
+        self.api_key = api_key or os.getenv("GOOGLE_API_KEY")
+        if not self.api_key:
+            logger.error("API key not provided")
+            raise ValueError("API key not provided - set GOOGLE_API_KEY environment variable or pass when initializing")
+        
+        # Configure the Gemini client
+        logger.info("Initializing Gemini API client")
+        genai.configure(api_key=self.api_key)
+        logger.info("Gemini API client initialized successfully")
+        
+    def send_usage_webhook(self, user_email: str, prompt: str) -> None:
+        """
+        Send usage data to webhook for tracking purposes
+        
+        Args:
+            user_email: Email of the user who processed the video
+            prompt: The prompt used for processing
+        """
+        try:
+            current_datetime = datetime.datetime.now().isoformat()
+            
+            webhook_data = {
+                "tool": "VIDEOQUERY",
+                "date": current_datetime,
+                "user": user_email,
+                "model": "GEMINI",
+                "settings": "no settings",
+                "subTool": "no subTool",
+                "prompt": prompt,
+                "negativePrompt": "no NEGATIVE_PROMPT",
+                "image": "no image"
+            }
+            
+            logger.info(f"Sending usage data to webhook for user: {user_email}")
+            
+            response = requests.post(
+                self.WEBHOOK_URL,
+                headers={"Content-Type": "application/json"},
+                data=json.dumps(webhook_data),
+                timeout=10  # 10 second timeout
+            )
+            
+            if response.status_code == 200:
+                logger.info("Successfully sent usage data to webhook")
+            else:
+                logger.warning(f"Webhook request failed with status code: {response.status_code}")
+                logger.warning(f"Response: {response.text}")
+                
+        except Exception as e:
+            logger.error(f"Error sending usage data to webhook: {str(e)}")
+            # Don't raise the exception - webhook failure shouldn't block the main flow
+    
+    def process_video(self, video_path: str, prompt: str, user_email: str = "anonymous") -> Dict[str, Any]:
+        """
+        Process a video with the given prompt using Gemini API
+        
+        Args:
+            video_path: Path to the video file
+            prompt: Text prompt to use for video analysis
+            user_email: Email of the user processing the video (for usage tracking)
+            
+        Returns:
+            Dictionary with processing result or error
+        """
+        result = {
+            "success": False,
+            "message": "",
+            "content": ""
+        }
+        
+        logger.info(f"Processing video: {video_path}")
+        logger.info(f"Prompt: {prompt[:100]}..." if len(prompt) > 100 else f"Prompt: {prompt}")
+        
+        if not os.path.exists(video_path):
+            error_msg = f"Video file not found at '{video_path}'"
+            logger.error(error_msg)
+            result["message"] = error_msg
+            return result
+        
+        try:
+            # Get file size
+            file_size = os.path.getsize(video_path)
+            logger.info(f"File size: {file_size / (1024 * 1024):.2f} MB")
+            
+            # Upload the video file
+            logger.info("Uploading video to Gemini API...")
+            
+            # Log the file size in relation to our threshold (for informational purposes only)
+            if file_size > self.CHUNKED_UPLOAD_THRESHOLD:
+                logger.info(f"File size exceeds {self.CHUNKED_UPLOAD_THRESHOLD/(1024*1024):.2f} MB threshold")
+            else:
+                logger.info(f"File size below {self.CHUNKED_UPLOAD_THRESHOLD/(1024*1024):.2f} MB threshold")
+            
+            # All uploads use the same method (our chunking happens in the frontend)
+            # Google API may handle large files internally in their own way
+            video_file = genai.upload_file(
+                path=video_path,
+                display_name=os.path.basename(video_path)
+            )
+            logger.info(f"Upload successful. File URI: {video_file.uri}")
+            logger.info(f"Initial file state: {video_file.state.name}")
+            
+            # Wait for processing if needed
+            processing_wait_count = 0
+            while video_file.state.name == "PROCESSING":
+                processing_wait_count += 1
+                logger.info(f"File is still processing. Wait count: {processing_wait_count}")
+                time.sleep(2)  # Wait for 2 seconds before checking again
+                video_file = genai.get_file(name=video_file.name)  # Re-fetch file state
+                logger.info(f"Updated file state: {video_file.state.name}")
+            
+            if video_file.state.name != "ACTIVE":
+                error_msg = f"Error: File did not become active. Current state: {video_file.state.name}"
+                logger.error(error_msg)
+                result["message"] = error_msg
+                return result
+            
+            # Determine MIME type for the video
+            mime_type, _ = mimetypes.guess_type(video_path)
+            if not mime_type:
+                logger.info(f"Could not determine MIME type, using default: video/mp4")
+                mime_type = "video/mp4"  # Fallback
+            else:
+                logger.info(f"MIME type: {mime_type}")
+            
+            # Create the content parts for the prompt
+            prompt_parts = [
+                {"text": prompt},
+                {"file_data": {
+                    "file_uri": video_file.uri,
+                    "mime_type": mime_type
+                }}
+            ]
+            
+            # Initialize the model and generate content
+            logger.info("Initializing GenerativeModel...")
+            model = genai.GenerativeModel(model_name="gemini-2.5-pro")
+            
+            logger.info("Sending prompt to Gemini for processing...")
+            response = model.generate_content(prompt_parts)
+            logger.info("Received response from Gemini")
+            
+            # Extract the response content
+            content = ""
+            if response.parts:
+                logger.info(f"Response has {len(response.parts)} parts")
+                for i, part in enumerate(response.parts):
+                    if hasattr(part, 'text'):
+                        part_text = part.text
+                        content_preview = part_text[:100] + '...' if len(part_text) > 100 else part_text
+                        logger.info(f"Part {i} (text): {content_preview}")
+                        content += part_text
+                    else:
+                        logger.info(f"Part {i} (no text): {type(part)}")
+            else:
+                logger.warning("No parts in response")
+                if hasattr(response, 'prompt_feedback') and response.prompt_feedback:
+                    logger.warning(f"Prompt feedback: {response.prompt_feedback}")
+            
+            # Set success result
+            result["success"] = True
+            result["content"] = content
+            logger.info(f"Processed result with {len(content)} characters")
+            
+            # Send usage data to webhook for tracking
+            self.send_usage_webhook(user_email, prompt)
+            
+            # Attempt to delete the file from Gemini storage
+            try:
+                logger.info(f"Deleting file from Gemini storage: {video_file.name}")
+                genai.delete_file(name=video_file.name)
+                logger.info("File deleted successfully from Gemini storage")
+            except Exception as del_err:
+                logger.warning(f"Could not delete file from Gemini storage: {str(del_err)}")
+            
+            return result
+            
+        except Exception as e:
+            import traceback
+            error_details = traceback.format_exc()
+            logger.error(f"Error processing video: {str(e)}")
+            logger.error(error_details)
+            result["message"] = f"Error processing video: {str(e)}"
+            result["error_details"] = error_details
+            return result
\ No newline at end of file
diff --git a/extract_user_logs.sh b/extract_user_logs.sh
new file mode 100755
index 0000000..42d461f
--- /dev/null
+++ b/extract_user_logs.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Script to extract user emails and prompts from veo-video-generator systemd logs
+# Usage: ./extract_user_logs.sh [output_file.csv]
+
+# Set default output file if not provided
+OUTPUT_FILE="${1:-video_generation_usage.csv}"
+
+# Service name (adjust if different)
+SERVICE_NAME="veo-video-generator"
+
+# Temporary file for processing
+TEMP_FILE=$(mktemp)
+
+echo "Extracting logs from systemd service: $SERVICE_NAME"
+echo "Output file: $OUTPUT_FILE"
+
+# Create CSV header
+echo "timestamp,user_email,prompt,video_length_sec,aspect_ratio,person_generation" > "$OUTPUT_FILE"
+
+# Extract logs from journalctl and process them
+journalctl -u "$SERVICE_NAME" --no-pager --output=short-iso | \
+    grep "Raw JSON data received:" | \
+    while IFS= read -r line; do
+        # Extract timestamp (everything before the hostname)
+        timestamp=$(echo "$line" | awk '{print $1}')
+        
+        # Extract the JSON part (everything after "Raw JSON data received: ")
+        json_part=$(echo "$line" | sed -n "s/.*Raw JSON data received: \(.*\)/\1/p")
+        
+        # Check if we got valid JSON
+        if [ -n "$json_part" ]; then
+            # Use jq to parse JSON and extract fields
+            # Handle case where jq might fail on malformed JSON
+            user_email=$(echo "$json_part" | jq -r '.user_email // "N/A"' 2>/dev/null)
+            prompt=$(echo "$json_part" | jq -r '.prompt // "N/A"' 2>/dev/null | sed 's/,/;/g' | sed 's/"/\\"/g')
+            video_length=$(echo "$json_part" | jq -r '.video_length_sec // "N/A"' 2>/dev/null)
+            aspect_ratio=$(echo "$json_part" | jq -r '.aspect_ratio // "N/A"' 2>/dev/null)
+            person_generation=$(echo "$json_part" | jq -r '.person_generation // "N/A"' 2>/dev/null)
+            
+            # Only add to CSV if we successfully extracted data
+            if [ "$user_email" != "null" ] && [ "$user_email" != "N/A" ] && [ "$user_email" != "" ]; then
+                echo "\"$timestamp\",\"$user_email\",\"$prompt\",\"$video_length\",\"$aspect_ratio\",\"$person_generation\"" >> "$OUTPUT_FILE"
+            fi
+        fi
+    done
+
+# Count total records
+record_count=$(wc -l < "$OUTPUT_FILE")
+record_count=$((record_count - 1))  # Subtract header row
+
+echo "Processing complete!"
+echo "Total records extracted: $record_count"
+echo "Output saved to: $OUTPUT_FILE"
+
+# Show summary of unique users
+echo ""
+echo "Unique users found:"
+if [ $record_count -gt 0 ]; then
+    tail -n +2 "$OUTPUT_FILE" | cut -d',' -f2 | sort | uniq -c | sort -nr
+else
+    echo "No records found."
+fi
+
+# Clean up temp file
+rm -f "$TEMP_FILE"
\ No newline at end of file
diff --git a/extract_user_logs_robust.sh b/extract_user_logs_robust.sh
new file mode 100755
index 0000000..eda5c09
--- /dev/null
+++ b/extract_user_logs_robust.sh
@@ -0,0 +1,175 @@
+#!/bin/bash
+
+# Enhanced script to extract user emails and prompts from veo-video-generator systemd logs
+# Usage: ./extract_user_logs_robust.sh [output_file.csv] [service_name] [date_range]
+# Examples:
+#   ./extract_user_logs_robust.sh usage_report.csv
+#   ./extract_user_logs_robust.sh usage_report.csv veo-video-generator "--since=2024-06-01"
+
+# Set defaults
+OUTPUT_FILE="${1:-video_generation_usage.csv}"
+SERVICE_NAME="${2:-veo-video-generator}"
+DATE_RANGE="${3:-}"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Function to print colored output
+print_status() {
+    echo -e "${GREEN}[INFO]${NC} $1"
+}
+
+print_warning() {
+    echo -e "${YELLOW}[WARN]${NC} $1"
+}
+
+print_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Check if required tools are installed
+check_dependencies() {
+    local missing_deps=()
+    
+    if ! command -v journalctl >/dev/null 2>&1; then
+        missing_deps+=("systemd (journalctl)")
+    fi
+    
+    if ! command -v jq >/dev/null 2>&1; then
+        missing_deps+=("jq")
+    fi
+    
+    if [ ${#missing_deps[@]} -ne 0 ]; then
+        print_error "Missing required dependencies: ${missing_deps[*]}"
+        print_error "Please install missing dependencies and try again"
+        exit 1
+    fi
+}
+
+# Function to validate JSON and extract fields safely
+extract_json_fields() {
+    local json_string="$1"
+    local timestamp="$2"
+    
+    # Try to validate JSON first
+    if echo "$json_string" | jq empty 2>/dev/null; then
+        # Extract fields using jq
+        local user_email=$(echo "$json_string" | jq -r '.user_email // empty' 2>/dev/null)
+        local prompt=$(echo "$json_string" | jq -r '.prompt // empty' 2>/dev/null)
+        local video_length=$(echo "$json_string" | jq -r '.video_length_sec // empty' 2>/dev/null)
+        local aspect_ratio=$(echo "$json_string" | jq -r '.aspect_ratio // empty' 2>/dev/null)
+        local person_generation=$(echo "$json_string" | jq -r '.person_generation // empty' 2>/dev/null)
+        
+        # Clean up prompt for CSV (replace commas and quotes)
+        prompt=$(echo "$prompt" | sed 's/,/;/g' | sed 's/"/\\"/g')
+        
+        # Only output if we have essential fields
+        if [ -n "$user_email" ] && [ "$user_email" != "null" ]; then
+            echo "\"$timestamp\",\"$user_email\",\"$prompt\",\"$video_length\",\"$aspect_ratio\",\"$person_generation\""
+            return 0
+        fi
+    else
+        print_warning "Invalid JSON found at $timestamp: $json_string"
+    fi
+    
+    return 1
+}
+
+print_status "Starting log extraction..."
+print_status "Service: $SERVICE_NAME"
+print_status "Output file: $OUTPUT_FILE"
+if [ -n "$DATE_RANGE" ]; then
+    print_status "Date range: $DATE_RANGE"
+fi
+
+# Check dependencies
+check_dependencies
+
+# Check if service exists
+if ! systemctl list-units --full -a | grep -q "$SERVICE_NAME.service"; then
+    print_warning "Service '$SERVICE_NAME' not found in systemctl list-units"
+    print_warning "This might be normal if the service is not currently loaded"
+fi
+
+# Create CSV header
+echo "timestamp,user_email,prompt,video_length_sec,aspect_ratio,person_generation" > "$OUTPUT_FILE"
+
+# Build journalctl command
+JOURNAL_CMD="journalctl -u $SERVICE_NAME --no-pager --output=short-iso"
+if [ -n "$DATE_RANGE" ]; then
+    JOURNAL_CMD="$JOURNAL_CMD $DATE_RANGE"
+fi
+
+print_status "Extracting logs... (this may take a while for large log files)"
+
+# Counter for processing
+processed_lines=0
+valid_records=0
+
+# Process logs
+eval "$JOURNAL_CMD" | grep "Raw JSON data received:" | while IFS= read -r line; do
+    processed_lines=$((processed_lines + 1))
+    
+    # Show progress every 100 lines
+    if [ $((processed_lines % 100)) -eq 0 ]; then
+        print_status "Processed $processed_lines log lines..."
+    fi
+    
+    # Extract timestamp (first field)
+    timestamp=$(echo "$line" | awk '{print $1}')
+    
+    # Extract JSON part - handle various formats
+    json_part=""
+    if [[ "$line" =~ Raw\ JSON\ data\ received:\ (.+)$ ]]; then
+        json_part="${BASH_REMATCH[1]}"
+    else
+        # Fallback extraction method
+        json_part=$(echo "$line" | sed -n "s/.*Raw JSON data received: \(.*\)/\1/p")
+    fi
+    
+    # Process if we found JSON
+    if [ -n "$json_part" ]; then
+        if extract_json_fields "$json_part" "$timestamp" >> "$OUTPUT_FILE"; then
+            valid_records=$((valid_records + 1))
+        fi
+    fi
+done
+
+# Get final counts (need to do this outside the while loop due to subshell)
+record_count=$(tail -n +2 "$OUTPUT_FILE" | wc -l)
+
+print_status "Processing complete!"
+print_status "Total valid records extracted: $record_count"
+print_status "Output saved to: $OUTPUT_FILE"
+
+if [ $record_count -eq 0 ]; then
+    print_warning "No records found. This could mean:"
+    print_warning "  - No logs exist for the specified service/date range"
+    print_warning "  - The log format has changed"
+    print_warning "  - The service name is incorrect"
+    exit 1
+fi
+
+# Show summary statistics
+echo ""
+print_status "=== SUMMARY REPORT ==="
+
+# Unique users
+echo "Unique users found:"
+tail -n +2 "$OUTPUT_FILE" | cut -d',' -f2 | sed 's/"//g' | sort | uniq -c | sort -nr
+
+# Date range
+echo ""
+echo "Date range of requests:"
+tail -n +2 "$OUTPUT_FILE" | cut -d',' -f1 | sed 's/"//g' | sort | head -1 | xargs -I {} echo "First: {}"
+tail -n +2 "$OUTPUT_FILE" | cut -d',' -f1 | sed 's/"//g' | sort | tail -1 | xargs -I {} echo "Last: {}"
+
+# Most active users
+echo ""
+echo "Top 5 most active users:"
+tail -n +2 "$OUTPUT_FILE" | cut -d',' -f2 | sed 's/"//g' | sort | uniq -c | sort -nr | head -5
+
+print_status "Report generation complete!"
\ No newline at end of file
diff --git a/frontend/.env b/frontend/.env
new file mode 100644
index 0000000..ddaae9b
--- /dev/null
+++ b/frontend/.env
@@ -0,0 +1,2 @@
+PUBLIC_URL=/video_query
+REACT_APP_BASE_URL=/video_query
\ No newline at end of file
diff --git a/frontend/README.md b/frontend/README.md
new file mode 100644
index 0000000..f2099a3
--- /dev/null
+++ b/frontend/README.md
@@ -0,0 +1,54 @@
+# Video Query Frontend
+
+This is the frontend for the Video Query application.
+
+## Development
+
+```bash
+npm install
+npm start
+```
+
+## Building for Production
+
+To build the application for deployment at the `/video_query` path:
+
+```bash
+# Option 1: Use the build script (recommended)
+./build.sh
+
+# Option 2: Manual build with correct PUBLIC_URL
+PUBLIC_URL=/video_query npm run build
+```
+
+## Deployment
+
+After building, deploy the contents of the `build` directory to your web server:
+
+```bash
+# Example deployment command
+scp -r build/* user@server:/var/www/html/video_query/
+```
+
+## Troubleshooting
+
+### Static Assets Not Loading
+
+If you see errors like:
+```
+Loading failed for the