From 77494488d516a2c95359f07d4e59d6f856464bfa Mon Sep 17 00:00:00 2001
From: SamoilenkoVadym <samoylenko.vadym@gmail.comm>
Date: Mon, 9 Feb 2026 15:40:11 +0000
Subject: [PATCH] feat(deploy): add auto git pull and cleanup to deployment
 script

- Auto pull latest code before deployment
- Clean old Docker images before building new ones
- Add comprehensive cleanup commands documentation
- Add production deployment guide
- Fix frontend serving from /var/www/html for production
- Preserve build cache for faster deployments

Co-Authored-By: Claude Sonnet 4.5 (1M context) <noreply@anthropic.com>
---
 CLEANUP-COMMANDS.md        | 167 +++++++++++++++++++++++
 PRODUCTION-DEPLOY.md       | 264 +++++++++++++++++++++++++++++++++++++
 backend/app/main.py        |   4 +-
 deploy.sh                  |  92 +++++++++++--
 docker-compose.fastapi.yml |   8 +-
 5 files changed, 518 insertions(+), 17 deletions(-)
 create mode 100644 CLEANUP-COMMANDS.md
 create mode 100644 PRODUCTION-DEPLOY.md

diff --git a/CLEANUP-COMMANDS.md b/CLEANUP-COMMANDS.md
new file mode 100644
index 0000000..70d0a3f
--- /dev/null
+++ b/CLEANUP-COMMANDS.md
@@ -0,0 +1,167 @@
+# Server Cleanup Commands
+
+Before deploying a new version, you can use these commands to completely clean up old builds and free disk space.
+
+## 🧹 Complete Cleanup (Nuclear Option)
+
+Run these commands on the Ubuntu server **before** running `deploy.sh`:
+
+```bash
+# Navigate to project directory
+cd /opt/solventum-image-metadata
+
+# Stop all running containers
+sudo docker-compose -f docker-compose.fastapi.yml down --remove-orphans
+
+# Remove ALL Oliver Metadata related containers (including stopped ones)
+sudo docker ps -a | grep -E "oliver|solventum-image-metadata" | awk '{print $1}' | xargs -r sudo docker rm -f
+
+# Remove ALL Oliver Metadata related images
+sudo docker images | grep -E "oliver|solventum-image-metadata" | awk '{print $3}' | xargs -r sudo docker rmi -f
+
+# Remove ALL Oliver Metadata related volumes (⚠️ WARNING: This deletes database data!)
+sudo docker volume ls | grep oliver | awk '{print $2}' | xargs -r sudo docker volume rm
+
+# Clean Docker build cache
+sudo docker builder prune -af
+
+# Remove dangling images
+sudo docker image prune -af
+
+# Remove unused networks
+sudo docker network prune -f
+
+# Remove stopped containers
+sudo docker container prune -f
+```
+
+## 🗑️ Safe Cleanup (Keeps Database & Uploads)
+
+If you want to keep your database and uploaded files:
+
+```bash
+cd /opt/solventum-image-metadata
+
+# Stop containers
+sudo docker-compose -f docker-compose.fastapi.yml down
+
+# Remove only old images (not volumes)
+sudo docker images | grep -E "oliver|solventum-image-metadata" | awk '{print $3}' | xargs -r sudo docker rmi -f
+
+# Clean build cache (keep last 24 hours)
+sudo docker builder prune -f --filter "until=24h"
+
+# Clean system
+sudo docker system prune -f
+```
+
+## 📊 Check Disk Space
+
+```bash
+# Before cleanup
+df -h /var/lib/docker
+
+# Check Docker disk usage
+sudo docker system df
+
+# After cleanup
+sudo docker system df
+```
+
+## 🔍 Verify Cleanup
+
+```bash
+# Should return no Oliver containers
+sudo docker ps -a | grep -E "oliver|solventum"
+
+# Should return no Oliver images
+sudo docker images | grep -E "oliver|solventum"
+
+# List remaining volumes (should see redis-data if you kept volumes)
+sudo docker volume ls | grep oliver
+```
+
+## 🚀 Full Deployment Workflow
+
+Complete workflow for a fresh deployment:
+
+```bash
+# 1. Navigate to project
+cd /opt/solventum-image-metadata
+
+# 2. OPTIONAL: Backup database (recommended)
+sudo cp backend/data/oliver_metadata.db backend/data/oliver_metadata.db.backup-$(date +%Y%m%d-%H%M%S)
+
+# 3. Run safe cleanup
+sudo docker-compose -f docker-compose.fastapi.yml down
+sudo docker images | grep -E "oliver|solventum" | awk '{print $3}' | xargs -r sudo docker rmi -f
+sudo docker system prune -f
+
+# 4. Run deployment script (includes git pull)
+sudo ./deploy.sh
+```
+
+## ⚠️ WARNING: Data Loss Commands
+
+These commands will **PERMANENTLY DELETE** your data:
+
+```bash
+# Delete database (cannot be recovered unless backed up)
+sudo rm -rf /opt/solventum-image-metadata/backend/data/oliver_metadata.db
+
+# Delete all uploads (cannot be recovered)
+sudo rm -rf /opt/solventum-image-metadata/backend/uploads/*
+
+# Delete all volumes (includes Redis data)
+sudo docker volume rm $(sudo docker volume ls | grep oliver | awk '{print $2}')
+
+# Delete all frontend files
+sudo rm -rf /var/www/html/solventum-image-metadata/*
+```
+
+## 🔧 Troubleshooting
+
+### "Device or resource busy" error
+
+If you get errors removing images/containers:
+
+```bash
+# Force stop all Docker processes
+sudo systemctl stop docker
+sudo systemctl start docker
+
+# Then retry cleanup
+sudo docker system prune -af --volumes
+```
+
+### "Cannot remove container" error
+
+```bash
+# Find and kill process
+sudo docker ps -a | grep oliver
+sudo docker rm -f <container_id>
+
+# If still stuck, restart Docker
+sudo systemctl restart docker
+```
+
+### Check what's using disk space
+
+```bash
+# Largest Docker images
+sudo docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | sort -k 3 -h
+
+# Disk usage by container
+sudo docker ps -s
+
+# Build cache size
+sudo docker builder du
+```
+
+## 📝 Notes
+
+- The `deploy.sh` script now includes automatic cleanup
+- Old images are removed automatically during deployment
+- Build cache is preserved for faster builds (24 hour window)
+- Database and uploads are preserved unless explicitly deleted
+- Frontend files in `/var/www/html/` are backed up to `/tmp/` during deployment
diff --git a/PRODUCTION-DEPLOY.md b/PRODUCTION-DEPLOY.md
new file mode 100644
index 0000000..2e0e1f0
--- /dev/null
+++ b/PRODUCTION-DEPLOY.md
@@ -0,0 +1,264 @@
+# Production Deployment Guide
+
+Quick guide for deploying Oliver Metadata Tool v4.0 to Ubuntu server.
+
+## 📋 Prerequisites
+
+1. **Server Setup:**
+   - Ubuntu 20.04+ server
+   - Docker & Docker Compose installed
+   - Node.js 18+ & npm installed
+   - Apache/Nginx configured as reverse proxy
+
+2. **Required Files:**
+   - `.env` file in project root with production values
+   - Apache/Nginx config for reverse proxy
+
+3. **Repository Location:**
+   - Clone to: `/opt/solventum-image-metadata/`
+   - Frontend serves from: `/var/www/html/solventum-image-metadata/`
+
+## 🚀 Quick Deployment
+
+### First-Time Setup
+
+```bash
+# 1. Clone repository
+cd /opt
+sudo git clone <repository-url> solventum-image-metadata
+cd solventum-image-metadata
+
+# 2. Create .env file
+sudo cp .env.production .env
+sudo nano .env  # Edit with production values
+
+# 3. Configure frontend volume in docker-compose
+sudo nano docker-compose.fastapi.yml
+# Comment out line 69: - ./frontend/dist:/app/frontend/dist:ro
+
+# 4. Run deployment
+sudo ./deploy.sh
+```
+
+### Subsequent Updates
+
+```bash
+# Just run the deploy script - it handles everything!
+cd /opt/solventum-image-metadata
+sudo ./deploy.sh
+```
+
+The script automatically:
+- ✅ Pulls latest code from git
+- ✅ Cleans old Docker images
+- ✅ Builds new containers
+- ✅ Initializes database (first run only)
+- ✅ Builds React frontend
+- ✅ Deploys frontend to `/var/www/html/`
+- ✅ Runs health checks
+
+## 🧹 Clean Deployment (Remove Old Builds)
+
+If you need to completely clean up before deploying:
+
+```bash
+cd /opt/solventum-image-metadata
+
+# Option 1: Quick cleanup (recommended)
+sudo docker-compose -f docker-compose.fastapi.yml down
+sudo docker images | grep -E "oliver|solventum" | awk '{print $3}' | xargs -r sudo docker rmi -f
+sudo docker system prune -f
+
+# Option 2: Nuclear cleanup (see CLEANUP-COMMANDS.md)
+# Use only if you want to delete everything including database
+```
+
+Then run `sudo ./deploy.sh`
+
+## ⚙️ Configuration Files
+
+### `.env` File (Production)
+
+Required environment variables:
+
+```bash
+# OpenAI (required for AI features)
+OPENAI_API_KEY=sk-proj-...
+AI_MODEL=gpt-5.2
+
+# Azure AD SSO
+AZURE_TENANT_ID=e519c2e6-bc6d-4fdf-8d9c-923c2f002385
+AZURE_CLIENT_ID=9079054c-9620-4757-a256-23413042f1ef
+AZURE_CLIENT_SECRET=your-secret-here
+REDIRECT_URI=https://ai-sandbox.oliver.solutions/solventum-image-metadata/
+
+# Security
+SECRET_KEY=your-production-secret-key-here
+
+# Backend
+BACKEND_PORT=5001
+DEBUG=false
+```
+
+### Apache Virtual Host Example
+
+```apache
+<Location /solventum-image-metadata/api>
+    ProxyPass http://localhost:5001
+    ProxyPassReverse http://localhost:5001
+</Location>
+
+<Location /solventum-image-metadata/auth>
+    ProxyPass http://localhost:5001/auth
+    ProxyPassReverse http://localhost:5001/auth
+</Location>
+
+# Serve frontend static files
+Alias /solventum-image-metadata /var/www/html/solventum-image-metadata
+<Directory /var/www/html/solventum-image-metadata>
+    Options -Indexes +FollowSymLinks
+    AllowOverride None
+    Require all granted
+
+    # React Router support
+    RewriteEngine On
+    RewriteBase /solventum-image-metadata/
+    RewriteRule ^index\.html$ - [L]
+    RewriteCond %{REQUEST_FILENAME} !-f
+    RewriteCond %{REQUEST_FILENAME} !-d
+    RewriteRule . /solventum-image-metadata/index.html [L]
+</Directory>
+```
+
+## 🔍 Post-Deployment Verification
+
+```bash
+# 1. Check Docker containers
+sudo docker ps | grep oliver
+
+# 2. Check backend health
+curl http://localhost:5001/health
+
+# 3. Check API docs
+curl http://localhost:5001/docs
+
+# 4. Check frontend files
+ls -lh /var/www/html/solventum-image-metadata/
+
+# 5. View logs
+cd /opt/solventum-image-metadata
+sudo docker-compose -f docker-compose.fastapi.yml logs -f backend
+```
+
+## 🔧 Useful Commands
+
+```bash
+# View deployment logs
+cd /opt/solventum-image-metadata
+sudo docker-compose -f docker-compose.fastapi.yml logs -f
+
+# Restart backend only
+sudo docker-compose -f docker-compose.fastapi.yml restart backend
+
+# Stop all services
+sudo docker-compose -f docker-compose.fastapi.yml down
+
+# Start services
+sudo docker-compose -f docker-compose.fastapi.yml up -d
+
+# Access Redis CLI
+sudo docker exec -it oliver-redis redis-cli
+
+# Check database
+sudo ls -lh /opt/solventum-image-metadata/backend/data/
+
+# Backup database
+sudo cp backend/data/oliver_metadata.db backend/data/oliver_metadata.db.backup-$(date +%Y%m%d)
+```
+
+## 🚨 Troubleshooting
+
+### Deployment fails with "Git pull failed"
+
+```bash
+cd /opt/solventum-image-metadata
+sudo git status
+sudo git stash  # If uncommitted changes
+sudo git pull origin main
+sudo ./deploy.sh
+```
+
+### Backend health check fails
+
+```bash
+# Check logs
+sudo docker-compose -f docker-compose.fastapi.yml logs backend
+
+# Common issues:
+# 1. OPENAI_API_KEY not set
+# 2. Redis not running
+# 3. Port 5001 already in use
+```
+
+### Frontend not loading
+
+```bash
+# Check files exist
+ls -lh /var/www/html/solventum-image-metadata/
+
+# Check permissions
+sudo chown -R www-data:www-data /var/www/html/solventum-image-metadata/
+sudo chmod -R 755 /var/www/html/solventum-image-metadata/
+
+# Check Apache config
+sudo apache2ctl -t
+sudo systemctl reload apache2
+```
+
+### "Docker build failed"
+
+```bash
+# Clean Docker completely
+sudo docker system prune -af --volumes
+sudo systemctl restart docker
+sudo ./deploy.sh
+```
+
+## 📊 Monitoring
+
+### Check disk space
+
+```bash
+# Docker disk usage
+sudo docker system df
+
+# Project disk usage
+du -sh /opt/solventum-image-metadata
+du -sh /var/www/html/solventum-image-metadata
+```
+
+### Check logs
+
+```bash
+# Backend logs (last 100 lines)
+cd /opt/solventum-image-metadata
+sudo docker-compose -f docker-compose.fastapi.yml logs --tail=100 backend
+
+# Follow logs in real-time
+sudo docker-compose -f docker-compose.fastapi.yml logs -f
+```
+
+## 🔒 Security Notes
+
+1. **Never commit .env files** with secrets to git
+2. **Use strong SECRET_KEY** in production
+3. **Backup database regularly** before updates
+4. **Use HTTPS** for production (configure in Apache/Nginx)
+5. **Review CORS settings** in backend/app/main.py if needed
+
+## 📞 Support
+
+For issues:
+1. Check logs: `docker-compose logs`
+2. Review [CLEANUP-COMMANDS.md](CLEANUP-COMMANDS.md) for cleanup options
+3. See [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) for detailed steps
diff --git a/backend/app/main.py b/backend/app/main.py
index e8700a7..ebbd18f 100644
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -76,7 +76,9 @@ app.include_router(import_api.router, prefix="/import", tags=["import"])
 
 
 # Serve React frontend
-STATIC_DIR = Path("/var/www/html/solventum-image-metadata")
+# Try production path first, fallback to local frontend/dist
+FRONTEND_DIR = os.getenv("FRONTEND_DIR", "/var/www/html/solventum-image-metadata")
+STATIC_DIR = Path(FRONTEND_DIR) if Path(FRONTEND_DIR).exists() else Path(__file__).parent.parent.parent / "frontend" / "dist"
 
 @app.get("/")
 async def root():
diff --git a/deploy.sh b/deploy.sh
index 80d902d..554bab4 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -6,9 +6,9 @@
 # Usage: sudo ./deploy.sh
 #
 # Prerequisites:
-# - Run 'git pull origin main' manually before this script
 # - Configure Apache/Nginx reverse proxy separately
 # - Ensure .env file is configured
+# - Git repository must be clean (no uncommitted changes)
 
 set -e
 
@@ -173,17 +173,91 @@ log_info "✓ Backend directory exists"
 log_success "All pre-flight checks passed"
 
 # -----------------------------------------------------------------------------
-# Git info (optional)
+# Pull latest code from Git
 # -----------------------------------------------------------------------------
+log_step "Pulling Latest Code"
+
 if command -v git &> /dev/null && [[ -d "$SCRIPT_DIR/.git" ]]; then
+    cd "$SCRIPT_DIR"
+
+    # Get current commit before pull
+    COMMIT_BEFORE=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+
+    # Check for uncommitted changes
+    if [[ -n $(git status --porcelain 2>/dev/null) ]]; then
+        log_warn "Uncommitted changes detected:"
+        git status --short
+        read -p "Continue with deployment? [y/N] " -n 1 -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            error_exit "Deployment cancelled by user"
+        fi
+    fi
+
+    # Stash any local changes (just in case)
+    log_info "Stashing local changes (if any)..."
+    git stash push -m "Auto-stash before deployment $(date +%Y%m%d-%H%M%S)" || true
+
+    # Pull latest code
+    log_info "Pulling from origin/main..."
+    git pull origin main || error_exit "Git pull failed"
+
+    # Get new commit info
     COMMIT_HASH=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
     COMMIT_MSG=$(git log -1 --pretty=format:"%s" 2>/dev/null || echo "unknown")
     COMMIT_DATE=$(git log -1 --pretty=format:"%ci" 2>/dev/null || echo "unknown")
-    log_info "Deploying commit: $COMMIT_HASH"
+
+    if [[ "$COMMIT_BEFORE" != "$COMMIT_HASH" ]]; then
+        log_success "Code updated: $COMMIT_BEFORE → $COMMIT_HASH"
+    else
+        log_info "Already up to date at commit: $COMMIT_HASH"
+    fi
+
     log_info "Commit message: $COMMIT_MSG"
     log_info "Commit date: $COMMIT_DATE"
+else
+    log_warn "Git not available or not a git repository"
+    COMMIT_HASH="unknown"
+    COMMIT_MSG="unknown"
+    COMMIT_DATE="unknown"
 fi
 
+log_success "Code ready for deployment"
+
+# -----------------------------------------------------------------------------
+# Clean old Docker resources
+# -----------------------------------------------------------------------------
+log_step "Cleaning Old Docker Resources"
+
+cd "$SCRIPT_DIR"
+
+# Stop old containers
+log_info "Stopping old containers..."
+$DOCKER_COMPOSE -f "$COMPOSE_FILE" down --remove-orphans || log_warn "No containers to stop"
+
+# Remove old images for this project (keep base images)
+log_info "Removing old project images..."
+OLD_IMAGES=$(docker images --filter "reference=solventum-image-metadata*" --filter "reference=*oliver*" -q 2>/dev/null || true)
+if [[ -n "$OLD_IMAGES" ]]; then
+    docker rmi -f $OLD_IMAGES 2>/dev/null || log_warn "Some images could not be removed (may be in use)"
+    log_success "Old images removed"
+else
+    log_info "No old images to remove"
+fi
+
+# Clean build cache (keep last 24 hours)
+log_info "Cleaning Docker build cache..."
+docker builder prune -f --filter "until=24h" > /dev/null 2>&1 || true
+
+# Remove unused networks
+log_info "Removing unused networks..."
+docker network prune -f > /dev/null 2>&1 || true
+
+# Show disk space saved
+log_info "Docker cleanup complete"
+
+log_success "Old resources cleaned"
+
 # -----------------------------------------------------------------------------
 # Build Docker containers
 # -----------------------------------------------------------------------------
@@ -191,22 +265,12 @@ log_step "Building Docker Containers"
 
 cd "$SCRIPT_DIR"
 
-# Pull latest base images and build
+# Pull latest base images and build (use cache for efficiency)
 log_info "Building containers with latest base images..."
 $DOCKER_COMPOSE -f "$COMPOSE_FILE" build --pull || error_exit "Docker build failed"
 
 log_success "Docker containers built successfully"
 
-# -----------------------------------------------------------------------------
-# Stop existing services (graceful shutdown)
-# -----------------------------------------------------------------------------
-log_step "Stopping Existing Services"
-
-log_info "Stopping containers gracefully..."
-$DOCKER_COMPOSE -f "$COMPOSE_FILE" down --remove-orphans || log_warn "No existing containers to stop"
-
-log_success "Existing services stopped"
-
 # -----------------------------------------------------------------------------
 # Start Docker services
 # -----------------------------------------------------------------------------
diff --git a/docker-compose.fastapi.yml b/docker-compose.fastapi.yml
index 1ec4ddd..f02beb9 100644
--- a/docker-compose.fastapi.yml
+++ b/docker-compose.fastapi.yml
@@ -54,6 +54,9 @@ services:
       # Upload directory
       UPLOAD_DIR: /app/uploads
 
+      # Frontend directory (for serving static files)
+      FRONTEND_DIR: /app/frontend/dist
+
     volumes:
       # Persistent storage for uploads
       - ./backend/uploads:/app/uploads
@@ -61,8 +64,9 @@ services:
       - ./backend/data:/app/data
       # Persistent templates
       - ./backend/output:/app/output
-      # Frontend static files (for serving from FastAPI)
-      - /var/www/html/solventum-image-metadata:/var/www/html/solventum-image-metadata:ro
+      # Frontend static files (local dev only - on production, frontend is served by Apache/Nginx)
+      # Comment out the next line for production deployment:
+      - ./frontend/dist:/app/frontend/dist:ro
       # Excel lookup file (optional - comment out if file doesn't exist)
       # - ./Celum ID to Adobe Asset Path Mapping Spreadsheet (1).xlsx:/app/Celum ID to Adobe Asset Path Mapping Spreadsheet (1).xlsx:ro