#!/bin/bash

################################################################################
# AI-Impress Health Check & Alerting System
# Version: 1.0.0
# Purpose: Monitor system health and send alerts on problems
################################################################################

set -euo pipefail

# Configuration
# Get from Vault or fallback file
export VAULT_ADDR=http://127.0.0.1:8200
export VAULT_TOKEN=$(cat /opt/00-infrastructure/vault/.vault-token 2>/dev/null || echo "")

# Try to get Slack webhook from fallback file first
SLACK_CREDENTIALS="/opt/05-backups/scripts/.slack-webhook"
if [[ -f "$SLACK_CREDENTIALS" ]]; then
    source "$SLACK_CREDENTIALS"
fi

# Try Vault if not set
if [[ -z "$SLACK_WEBHOOK_URL" ]]; then
    SLACK_WEBHOOK_URL=$(vault kv get -field=slack_webhook secret/monitoring 2>/dev/null || echo "")
fi

ALERT_EMAIL="${ALERT_EMAIL:-admin@ai-impress.com}"
SMTP_SERVER="${SMTP_SERVER:-localhost}"
LOG_FILE="/opt/05-backups/logs/health-check-$(date +%Y%m%d).log"

# Thresholds
DISK_THRESHOLD=90  # Alert if disk > 90%
MEMORY_THRESHOLD=90  # Alert if memory > 90%
MAX_UNHEALTHY_CONTAINERS=2  # Alert if more than 2 containers unhealthy

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

PROBLEMS=()
WARNINGS=()

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

add_problem() {
    PROBLEMS+=("$1")
    log "🔴 PROBLEM: $1"
}

add_warning() {
    WARNINGS+=("$1")
    log "🟡 WARNING: $1"
}

################################################################################
# CHECKS
################################################################################

check_critical_services() {
    log "Checking critical services..."
    
    local critical_services=("traefik" "postgres-main" "redis-main")
    
    for service in "${critical_services[@]}"; do
        if ! docker ps --format '{{.Names}}' | grep -q "^${service}$"; then
            add_problem "Critical service $service is NOT RUNNING"
        else
            local health=$(docker inspect --format='{{.State.Health.Status}}' "$service" 2>/dev/null || echo "no healthcheck")
            if [[ "$health" == "unhealthy" ]]; then
                add_problem "Critical service $service is UNHEALTHY"
            fi
        fi
    done
}

check_websites() {
    log "Checking websites..."
    
    local websites=(
        "wiki.ai-impress.com"
        "n8n.ai-impress.com"
        "odoo.ai-impress.com"
        "auth.ai-impress.com"
    )
    
    for site in "${websites[@]}"; do
        local http_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 "https://$site" 2>/dev/null || echo "000")
        
        if [[ "$http_code" != "200" ]] && [[ ! "$http_code" =~ ^30 ]]; then
            add_problem "Website $site is DOWN (HTTP $http_code)"
        fi
    done
}

check_disk_space() {
    log "Checking disk space..."
    
    local disk_usage=$(df -h / | awk 'NR==2 {print $5}' | sed 's/%//')
    
    if [[ "$disk_usage" -gt "$DISK_THRESHOLD" ]]; then
        add_problem "Disk usage is CRITICAL: ${disk_usage}% (threshold: ${DISK_THRESHOLD}%)"
    elif [[ "$disk_usage" -gt 80 ]]; then
        add_warning "Disk usage is high: ${disk_usage}%"
    fi
    
    # Check /mnt/psql-data
    if [[ -d /mnt/psql-data ]]; then
        local db_disk_usage=$(df -h /mnt/psql-data | awk 'NR==2 {print $5}' | sed 's/%//')
        if [[ "$db_disk_usage" -gt "$DISK_THRESHOLD" ]]; then
            add_problem "Database disk usage is CRITICAL: ${db_disk_usage}%"
        fi
    fi
}

check_memory() {
    log "Checking memory usage..."
    
    local memory_usage=$(free | awk '/Mem:/ {printf "%.0f", ($3/$2)*100}')
    
    if [[ "$memory_usage" -gt "$MEMORY_THRESHOLD" ]]; then
        add_problem "Memory usage is CRITICAL: ${memory_usage}% (threshold: ${MEMORY_THRESHOLD}%)"
    elif [[ "$memory_usage" -gt 80 ]]; then
        add_warning "Memory usage is high: ${memory_usage}%"
    fi
}

check_unhealthy_containers() {
    log "Checking unhealthy containers..."

    local unhealthy_count=$(docker ps --filter "health=unhealthy" -q | wc -l)

    if [[ "$unhealthy_count" -gt "$MAX_UNHEALTHY_CONTAINERS" ]]; then
        local containers=$(docker ps --filter "health=unhealthy" --format '{{.Names}}' | tr '\n' ', ')
        add_problem "$unhealthy_count containers are UNHEALTHY: $containers"
    elif [[ "$unhealthy_count" -gt 0 ]]; then
        local containers=$(docker ps --filter "health=unhealthy" --format '{{.Names}}' | tr '\n' ', ')
        add_warning "$unhealthy_count container(s) unhealthy: $containers"
    fi
}

check_r2_usage() {
    log "Checking Cloudflare R2 backup storage..."

    # Load Restic environment
    if [[ ! -f /opt/05-backups/restic/.env ]]; then
        add_warning "Restic config not found - skipping R2 check"
        return
    fi

    source /opt/05-backups/restic/.env

    # Get R2 stats - use raw-data mode to get actual R2 storage usage (with deduplication)
    local r2_stats=$(restic stats --mode raw-data 2>/dev/null | grep "Total Size")

    if [[ -z "$r2_stats" ]]; then
        add_warning "Unable to get R2 statistics"
        return
    fi

    # Extract size in GiB and convert to GB
    local size_gib=$(echo "$r2_stats" | grep -oP '\d+\.\d+' | head -1)
    local size_gb=$(echo "$size_gib" | awk '{printf "%.2f", $1}')
    local r2_limit=10  # Cloudflare R2 free tier limit: 10 GB
    local usage_percent=$(echo "$size_gb $r2_limit" | awk '{printf "%.1f", ($1/$2)*100}')

    # Check if bc is available for floating point comparison
    if command -v bc &> /dev/null; then
        if (( $(echo "$size_gb > $r2_limit" | bc -l) )); then
            add_problem "R2 storage EXCEEDED: ${size_gb}GB / ${r2_limit}GB (${usage_percent}%)"
        elif (( $(echo "$size_gb > 7" | bc -l) )); then
            add_warning "R2 storage high: ${size_gb}GB / ${r2_limit}GB (${usage_percent}%, >70%)"
        else
            log "✅ R2 storage OK: ${size_gb}GB / ${r2_limit}GB (${usage_percent}%)"
        fi
    else
        # Fallback: use integer comparison if bc not available
        local size_gb_int=$(echo "$size_gb" | cut -d. -f1)
        if [[ "$size_gb_int" -gt "$r2_limit" ]]; then
            add_problem "R2 storage EXCEEDED: ${size_gb}GB / ${r2_limit}GB (${usage_percent}%)"
        elif [[ "$size_gb_int" -ge 7 ]]; then
            add_warning "R2 storage high: ${size_gb}GB / ${r2_limit}GB (${usage_percent}%, >70%)"
        else
            log "✅ R2 storage OK: ${size_gb}GB / ${r2_limit}GB (${usage_percent}%)"
        fi
    fi

    # Check snapshot count - using JSON for accurate count
    local snapshot_count=$(restic snapshots --json 2>/dev/null | python3 -c 'import sys, json; print(len(json.load(sys.stdin)))' 2>/dev/null || echo "0")
    log "📦 R2 snapshots: $snapshot_count (policy: keep 3 daily + 1 weekly)"

    if [[ $snapshot_count -gt 5 ]]; then
        add_warning "Too many R2 snapshots: $snapshot_count (expected ≤4)"
    fi
}

check_backup_status() {
    log "Checking backup status..."

    # Check Restic snapshots (primary backup method)
    if [[ -f /opt/05-backups/restic/.env ]]; then
        source /opt/05-backups/restic/.env

        # Get latest snapshot time using python to properly parse JSON array
        local latest_snapshot=$(restic snapshots --compact --json 2>/dev/null | python3 -c 'import sys, json; snapshots = json.load(sys.stdin); print(snapshots[-1]["time"] if snapshots else "")' 2>/dev/null)

        if [[ -z "$latest_snapshot" ]]; then
            add_problem "NO RESTIC SNAPSHOTS FOUND in R2 backup"
        else
            # Convert snapshot time to epoch
            local snapshot_epoch=$(date -d "$latest_snapshot" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "${latest_snapshot%%.*}" +%s 2>/dev/null)
            local current_epoch=$(date +%s)
            local backup_age_hours=$(( (current_epoch - snapshot_epoch) / 3600 ))
            local backup_age_days=$(( backup_age_hours / 24 ))

            if [[ "$backup_age_days" -gt 2 ]]; then
                add_problem "Restic backup is $backup_age_days days old (last: $(date -d "$latest_snapshot" '+%Y-%m-%d %H:%M' 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "${latest_snapshot%%.*}" '+%Y-%m-%d %H:%M' 2>/dev/null))"
            elif [[ "$backup_age_hours" -gt 36 ]]; then
                add_warning "Restic backup is ${backup_age_hours}h old"
            else
                log "✅ Restic backup OK: ${backup_age_hours}h old"
            fi
        fi
    else
        add_warning "Restic config not found - cannot check cloud backups"
    fi

    # Check local database backups as secondary check
    if [[ -d /mnt/backups/local-backups ]]; then
        local latest_db_backup=$(find /mnt/backups/local-backups -type f -name "postgresql-*.sql.gz" 2>/dev/null | sort | tail -1)

        if [[ -n "$latest_db_backup" ]]; then
            local db_backup_age_hours=$(( ($(date +%s) - $(stat -c %Y "$latest_db_backup")) / 3600 ))

            if [[ "$db_backup_age_hours" -gt 48 ]]; then
                add_warning "Local DB backup is $(($db_backup_age_hours / 24)) days old (last: $(basename "$latest_db_backup"))"
            else
                log "✅ Local DB backup OK: $(basename "$latest_db_backup") (${db_backup_age_hours}h old)"
            fi
        fi
    fi
}

check_container_restarts() {
    log "Checking for excessive container restarts..."

    while read -r container restart_count started_at; do
        # Calculate container uptime in hours
        local started_epoch=$(date -d "$started_at" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "${started_at%%.*}" +%s 2>/dev/null)
        local current_epoch=$(date +%s)
        local uptime_hours=$(( (current_epoch - started_epoch) / 3600 ))

        # Only alert if container has high restart count AND recently restarted (uptime < 24h)
        if [[ "$restart_count" -gt 10 ]] && [[ "$uptime_hours" -lt 24 ]]; then
            add_problem "Container $container has restarted $restart_count times (uptime: ${uptime_hours}h)"
        elif [[ "$restart_count" -gt 5 ]] && [[ "$uptime_hours" -lt 48 ]]; then
            add_warning "Container $container has restarted $restart_count times (uptime: ${uptime_hours}h)"
        fi
    done < <(docker ps --format '{{.Names}}' | xargs -I {} sh -c 'echo {} $(docker inspect --format="{{.RestartCount}} {{.State.StartedAt}}" {})')
}

################################################################################
# ALERT SENDING
################################################################################

send_slack_alert() {
    local message="$1"
    
    if [[ -z "$SLACK_WEBHOOK_URL" ]]; then
        log "Slack webhook not configured, skipping..."
        return
    fi
    
    local payload=$(cat <<EOF
{
    "username": "AI-Impress Monitor",
    "icon_emoji": ":warning:",
    "text": "$message"
}
EOF
)
    
    curl -X POST -H 'Content-type: application/json' \
        --data "$payload" \
        "$SLACK_WEBHOOK_URL" 2>/dev/null || log "Failed to send Slack alert"
}

send_email_alert() {
    local subject="$1"
    local body="$2"
    
    # Try to send via mail command if available
    if command -v mail &> /dev/null; then
        echo "$body" | mail -s "$subject" "$ALERT_EMAIL" 2>/dev/null || log "Failed to send email"
    elif command -v sendmail &> /dev/null; then
        echo -e "Subject: $subject\nTo: $ALERT_EMAIL\n\n$body" | sendmail "$ALERT_EMAIL" 2>/dev/null || log "Failed to send email"
    else
        log "No mail command available, saving email to file"
        echo -e "To: $ALERT_EMAIL\nSubject: $subject\n\n$body" > "/tmp/alert-email-$(date +%s).txt"
    fi
}

generate_alert_report() {
    local report="🚨 AI-Impress Server Health Alert\n\n"
    report+="Server: ai-impress-prod (51.89.231.46)\n"
    report+="Time: $(date '+%Y-%m-%d %H:%M:%S')\n\n"
    
    if [[ ${#PROBLEMS[@]} -gt 0 ]]; then
        report+="🔴 CRITICAL PROBLEMS (${#PROBLEMS[@]}):\n"
        for problem in "${PROBLEMS[@]}"; do
            report+="  - $problem\n"
        done
        report+="\n"
    fi
    
    if [[ ${#WARNINGS[@]} -gt 0 ]]; then
        report+="🟡 WARNINGS (${#WARNINGS[@]}):\n"
        for warning in "${WARNINGS[@]}"; do
            report+="  - $warning\n"
        done
        report+="\n"
    fi
    
    report+="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n"
    report+="🔧 HOW TO FIX:\n\n"
    report+="1. SSH to server:\n"
    report+="   ssh ubuntu@51.89.231.46\n\n"
    report+="2. Check full status:\n"
    report+="   /opt/05-backups/scripts/admin.sh status\n\n"
    report+="3. View detailed logs:\n"
    report+="   docker logs <container-name> --tail 100\n\n"
    report+="4. Restart service if needed:\n"
    report+="   docker restart <container-name>\n\n"
    report+="5. Check disk space:\n"
    report+="   df -h\n\n"
    report+="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n"
    report+="📊 Quick System Status:\n"
    report+="  Memory: $(free -h | awk '/^Mem:/ {print $3 "/" $2}')\n"
    report+="  Disk: $(df -h / | awk 'NR==2 {print $3 "/" $2 " (" $5 ")"}')\n"
    report+="  Containers: $(docker ps -q | wc -l) running\n"
    report+="  Uptime: $(uptime -p)\n\n"
    report+="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
    report+="Generated by: /opt/05-backups/scripts/health-check-alerting.sh\n"
    
    echo -e "$report"
}

################################################################################
# MAIN
################################################################################

main() {
    log "╔══════════════════════════════════════════════════════╗"
    log "║  AI-Impress Health Check & Alerting System          ║"
    log "║  $(date '+%Y-%m-%d %H:%M:%S')                            ║"
    log "╚══════════════════════════════════════════════════════╝"
    log ""
    
    # Run all checks
    check_critical_services
    check_websites
    check_disk_space
    check_memory
    check_r2_usage
    check_unhealthy_containers
    check_backup_status
    check_container_restarts
    
    log ""
    log "Summary: ${#PROBLEMS[@]} problems, ${#WARNINGS[@]} warnings"
    
    # Send alerts if there are problems
    if [[ ${#PROBLEMS[@]} -gt 0 ]]; then
        log "🚨 CRITICAL PROBLEMS DETECTED - Sending alerts..."
        
        local alert_report=$(generate_alert_report)
        
        # Send to Slack
        send_slack_alert "$alert_report"
        
        # Send via Email
        send_email_alert "🚨 AI-Impress Server Alert - ${#PROBLEMS[@]} Critical Problems" "$alert_report"
        
        echo -e "\n${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
        echo -e "${RED}⚠️  CRITICAL PROBLEMS DETECTED!${NC}"
        echo -e "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
        for problem in "${PROBLEMS[@]}"; do
            echo -e "${RED}  • $problem${NC}"
        done
        echo -e "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"
        
        exit 1
    elif [[ ${#WARNINGS[@]} -gt 0 ]]; then
        log "⚠️  Warnings detected (no critical problems)"

        local warning_report=$(generate_alert_report)

        # Send to Slack for warnings too
        send_slack_alert "$warning_report"

        echo -e "\n${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
        echo -e "${YELLOW}⚠️  WARNINGS DETECTED${NC}"
        echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
        for warning in "${WARNINGS[@]}"; do
            echo -e "${YELLOW}  • $warning${NC}"
        done
        echo -e "${YELLOW}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"

        exit 0
    else
        log "✅ All checks passed - System is healthy"
        echo -e "\n${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
        echo -e "${GREEN}✅ All checks passed - System is healthy${NC}"
        echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"
        
        exit 0
    fi
}

# Run if executed directly
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
    main "$@"
fi
