json-parser-twist-metaserver/OLD/json_workflow_processor-hybrid.py

#!/usr/bin/env python3
"""
JSON Workflow Processor - HYBRID VERSION
Real-time monitoring + periodic scanning + failed file handling

Features:
- File system events for immediate detection
- Periodic scanning every 60 seconds as backup
- Failed file handling (moved to failed folder)
- Startup scan for crash recovery
- Concurrent processing with detailed reporting
- Prevents double processing with file tracking
"""

import json
import logging
import smtplib
import shutil
import time
import threading
import schedule
from collections import defaultdict, Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, date, timedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
from pathlib import Path
from typing import Dict, Optional, Any, List, Set
from queue import Queue
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler


class Config:
    """Configuration settings"""
    
    # Paths
    HOT_FOLDER = Path("/data/PRODUCTION/JSON")
    JSON_STORE = Path("/data/PRODUCTION/JSON_STORE/")
    JSON_FAILED = Path("/data/PRODUCTION/JSON_FAILED/")
    SYNC_BASE = Path("/data/PRODUCTION/SYNC/MAKE")
    REPORTS_DIR = Path("/PRODUCTION/JSON_PARSER_LOGS")
    
    # Clients eligible for Celtra projects (based on JobCategory)
    CELTRA_ELIGIBLE_CLIENTS = {
        "CIBC", "OLIVER", "ADIDAS", "PAYPAL", 
        "RECKITTBENCKISER", "BAYER", "3M", "RANK"
    }
    
    # Client destinations
    DESTINATIONS = {
        "CELTRA": SYNC_BASE / "Celtra - Create_Rename - Project_Design File",
        "RECKITTBENCKISER": SYNC_BASE / "Monday RB", 
        "RANK": SYNC_BASE / "Monday Rank"
    }
    
    # Email settings
    SMTP_SERVER = "smtp.mailgun.org"
    SMTP_PORT = 587
    SMTP_USER = "twist@mail.dev.oliver.solutions"
    SMTP_PASSWORD = "102115e9f3b9d7332d0cd1d4329bc0d4-77751bfc-ca066b71"
    SENDER_EMAIL = "TWIST-UK-SERVER@oliver.agency"
    ERROR_EMAIL = "daveporter@oliver.agency"
    REPORT_EMAILS = ["daveporter@oliver.agency"]
    
    # Batch processing settings
    BATCH_SIZE = 10
    MAX_WORKERS = 10
    BATCH_TIMEOUT = 30
    
    # Monitoring settings
    POLL_INTERVAL = 2
    WAIT_DELAY = 3
    PERIODIC_SCAN_INTERVAL = 60  # Scan every 60 seconds for missed files
    
    # Reporting settings
    DAILY_REPORT_TIME = "00:00"
    KEEP_REPORTS_DAYS = 30


class DailyStats:
    """Thread-safe daily statistics collector"""
    
    def __init__(self):
        self.lock = threading.Lock()
        self.reset_stats()
        self.start_time = datetime.now()
    
    def reset_stats(self):
        """Reset daily statistics"""
        with self.lock:
            self.date = date.today()
            self.total_processed = 0
            self.total_errors = 0
            self.total_batches = 0
            self.startup_files_processed = 0
            self.periodic_files_found = 0
            self.failed_files_moved = 0
            
            # Client breakdown
            self.client_stats = defaultdict(lambda: {
                'processed': 0,
                'routed': 0,
                'celtra': 0,
                'errors': 0,
                'job_categories': Counter(),
                'studio_codes': Counter()
            })
            
            # Destination breakdown
            self.destination_stats = defaultdict(int)
            
            # Performance metrics
            self.processing_times = []
            self.batch_sizes = []
            
            # Error tracking
            self.error_details = []
            
            # Hourly breakdown
            self.hourly_stats = defaultdict(int)
    
    def record_file_processed(self, client: str, json_data: Dict[str, Any], 
                             destinations: List[Path], processing_time: float,
                             success: bool = True, error: str = None, 
                             is_startup: bool = False, is_periodic: bool = False):
        """Record a processed file"""
        with self.lock:
            hour = datetime.now().hour
            self.hourly_stats[hour] += 1
            
            if is_startup:
                self.startup_files_processed += 1
            elif is_periodic:
                self.periodic_files_found += 1
            
            if success:
                self.total_processed += 1
                self.client_stats[client]['processed'] += 1
                
                # Track job categories and studio codes
                job_category = json_data.get('JobCategory', 'Unknown')
                studio_code = json_data.get('StudioCode', 'Unknown')
                self.client_stats[client]['job_categories'][job_category] += 1
                self.client_stats[client]['studio_codes'][studio_code] += 1
                
                # Track routing
                if destinations:
                    self.client_stats[client]['routed'] += 1
                    for dest in destinations:
                        self.destination_stats[dest.name] += 1
                        if 'Celtra' in dest.name:
                            self.client_stats[client]['celtra'] += 1
                
                # Track performance
                self.processing_times.append(processing_time)
            else:
                self.total_errors += 1
                self.client_stats[client]['errors'] += 1
                if error:
                    self.error_details.append({
                        'time': datetime.now().strftime('%H:%M:%S'),
                        'client': client,
                        'error': error
                    })
    
    def record_failed_file_moved(self):
        """Record a failed file being moved"""
        with self.lock:
            self.failed_files_moved += 1
    
    def record_batch(self, batch_size: int):
        """Record a completed batch"""
        with self.lock:
            self.total_batches += 1
            self.batch_sizes.append(batch_size)
    
    def get_stats_summary(self) -> Dict[str, Any]:
        """Get current statistics summary"""
        with self.lock:
            avg_processing_time = sum(self.processing_times) / len(self.processing_times) if self.processing_times else 0
            avg_batch_size = sum(self.batch_sizes) / len(self.batch_sizes) if self.batch_sizes else 0
            
            return {
                'date': self.date.strftime('%Y-%m-%d'),
                'uptime': str(datetime.now() - self.start_time).split('.')[0],
                'total_processed': self.total_processed,
                'startup_files_processed': self.startup_files_processed,
                'periodic_files_found': self.periodic_files_found,
                'failed_files_moved': self.failed_files_moved,
                'total_errors': self.total_errors,
                'total_batches': self.total_batches,
                'success_rate': (self.total_processed / (self.total_processed + self.total_errors) * 100) if (self.total_processed + self.total_errors) > 0 else 0,
                'avg_processing_time': round(avg_processing_time, 2),
                'avg_batch_size': round(avg_batch_size, 1),
                'client_stats': dict(self.client_stats),
                'destination_stats': dict(self.destination_stats),
                'hourly_stats': dict(self.hourly_stats),
                'error_details': self.error_details.copy()
            }


class HybridProcessor:
    """JSON processor with hybrid monitoring (events + periodic scanning)"""
    
    def __init__(self):
        self.setup_logging()
        self.ensure_directories()
        self.file_queue = Queue()
        self.daily_stats = DailyStats()
        self.startup_complete = False
        
        # Track recently processed files to avoid duplicates
        self.recently_processed: Set[str] = set()
        self.processed_lock = threading.Lock()
        
        self.setup_daily_reporting()
        
    def setup_logging(self):
        """Setup logging configuration"""
        log_format = '%(asctime)s - %(name)s - %(levelname)s - [%(threadName)s] %(message)s'
        log_dir = Path("/PRODUCTION/JSON_PARSER_LOGS")
        log_dir.mkdir(parents=True, exist_ok=True)
        
        logging.basicConfig(
            level=logging.INFO,
            format=log_format,
            handlers=[
                logging.FileHandler(log_dir / 'json_workflow_hybrid.log'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)
        self.logger.info("Hybrid JSON Workflow Processor initialized")
        
    def ensure_directories(self):
        """Create necessary directories"""
        directories = [
            Config.HOT_FOLDER,
            Config.JSON_STORE,
            Config.JSON_FAILED,
            Config.SYNC_BASE,
            Config.REPORTS_DIR
        ] + list(Config.DESTINATIONS.values())
        
        for directory in directories:
            directory.mkdir(parents=True, exist_ok=True)
            directory.chmod(0o777)
    
    def setup_daily_reporting(self):
        """Setup scheduled daily reporting"""
        schedule.every().day.at(Config.DAILY_REPORT_TIME).do(self.generate_daily_report)
        
        def run_scheduler():
            while True:
                schedule.run_pending()
                time.sleep(60)
        
        scheduler_thread = threading.Thread(target=run_scheduler, name="Scheduler", daemon=True)
        scheduler_thread.start()
    
    def add_to_recently_processed(self, file_path: Path):
        """Add file to recently processed list"""
        with self.processed_lock:
            # Use file path + size + mtime as unique identifier
            file_id = f"{file_path}_{file_path.stat().st_size}_{file_path.stat().st_mtime}"
            self.recently_processed.add(file_id)
            
            # Keep only last 1000 processed files in memory
            if len(self.recently_processed) > 1000:
                # Remove oldest 200 entries
                oldest_entries = list(self.recently_processed)[:200]
                for entry in oldest_entries:
                    self.recently_processed.discard(entry)
    
    def was_recently_processed(self, file_path: Path) -> bool:
        """Check if file was recently processed"""
        try:
            with self.processed_lock:
                file_id = f"{file_path}_{file_path.stat().st_size}_{file_path.stat().st_mtime}"
                return file_id in self.recently_processed
        except:
            return False
    
    def scan_existing_files(self) -> List[Path]:
        """Scan all client folders for existing JSON files"""
        existing_files = []
        
        try:
            self.logger.info("🔍 Scanning for existing JSON files...")
            
            if not Config.HOT_FOLDER.exists():
                self.logger.warning(f"Hot folder does not exist: {Config.HOT_FOLDER}")
                return existing_files
            
            # Scan all subdirectories for JSON files
            for client_folder in Config.HOT_FOLDER.iterdir():
                if client_folder.is_dir():
                    client_name = client_folder.name
                    json_files = list(client_folder.glob("*.json"))
                    
                    if json_files:
                        self.logger.info(f"📁 Found {len(json_files)} JSON files in {client_name}/")
                        existing_files.extend(json_files)
                    else:
                        self.logger.debug(f"📁 No JSON files in {client_name}/")
            
            self.logger.info(f"🔍 Startup scan complete: Found {len(existing_files)} existing JSON files")
            return existing_files
            
        except Exception as e:
            self.logger.error(f"Error during startup scan: {e}")
            return existing_files
    
    def periodic_scan(self):
        """Periodic scan for missed files"""
        while True:
            try:
                if self.startup_complete:
                    time.sleep(Config.PERIODIC_SCAN_INTERVAL)
                    
                    self.logger.debug("🔄 Periodic scan starting...")
                    missed_files = []
                    
                    # Scan for files that weren't caught by file system events
                    for client_folder in Config.HOT_FOLDER.iterdir():
                        if client_folder.is_dir():
                            for json_file in client_folder.glob("*.json"):
                                if not self.was_recently_processed(json_file):
                                    missed_files.append(json_file)
                    
                    if missed_files:
                        self.logger.info(f"🔄 Periodic scan found {len(missed_files)} missed files")
                        for file_path in missed_files:
                            self.queue_file(file_path, is_periodic=True)
                    else:
                        self.logger.debug("🔄 Periodic scan: no missed files")
                else:
                    time.sleep(5)  # Wait for startup to complete
                    
            except Exception as e:
                self.logger.error(f"Error during periodic scan: {e}")
                time.sleep(Config.PERIODIC_SCAN_INTERVAL)
    
    def move_failed_file(self, file_path: Path, error_msg: str):
        """Move failed file to failed folder"""
        try:
            # Determine client folder
            client_folder = file_path.parent.name if file_path.parent != Config.HOT_FOLDER else "UNKNOWN"
            
            # Create failed directory structure
            failed_client_dir = Config.JSON_FAILED / client_folder
            failed_client_dir.mkdir(parents=True, exist_ok=True)
            failed_client_dir.chmod(0o777)
            
            # Create timestamped filename to avoid conflicts
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            failed_filename = f"{timestamp}_{file_path.name}"
            failed_path = failed_client_dir / failed_filename
            
            # Move file to failed folder
            shutil.move(str(file_path), str(failed_path))
            failed_path.chmod(0o777)
            
            # Create error log file
            error_log_path = failed_client_dir / f"{timestamp}_{file_path.stem}_ERROR.txt"
            with open(error_log_path, 'w') as f:
                f.write(f"File: {file_path.name}\n")
                f.write(f"Error Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
                f.write(f"Error: {error_msg}\n")
            error_log_path.chmod(0o777)
            
            self.logger.warning(f"❌ Moved failed file: {file_path.name} → {failed_path}")
            self.daily_stats.record_failed_file_moved()
            
        except Exception as e:
            self.logger.error(f"Failed to move failed file {file_path.name}: {e}")
    
    def process_startup_files(self, files: List[Path]):
        """Process existing files found during startup scan"""
        if not files:
            self.logger.info("✅ No existing files to process")
            return
        
        self.logger.info(f"🚀 Processing {len(files)} existing files from startup scan...")
        
        # Process in batches
        batch_size = Config.BATCH_SIZE
        for i in range(0, len(files), batch_size):
            batch = files[i:i + batch_size]
            self.logger.info(f"Processing startup batch {i//batch_size + 1}: {len(batch)} files")
            self.process_batch(batch, is_startup=True)
        
        self.logger.info(f"✅ Startup file processing complete: {len(files)} files processed")
    
    def parse_json_file(self, file_path: Path) -> Optional[Dict[str, Any]]:
        """Parse JSON file and extract key fields"""
        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)
            
            job_details = data.get("JobSpecification", {}).get("JobDetails", {})
            
            extracted = {
                "JobCategory": job_details.get("JobCategory", ""),
                "StudioCode": job_details.get("StudioCode", ""),
                "Title": job_details.get("Title", ""),
                "ClientCode": job_details.get("ClientCode", "")
            }
            
            return extracted
            
        except json.JSONDecodeError as e:
            self.logger.error(f"JSON parsing error in {file_path.name}: {e}")
            return None
        except Exception as e:
            self.logger.error(f"Error reading {file_path.name}: {e}")
            return None
    
    def determine_routing(self, json_data: Dict[str, Any], client_folder: str) -> List[Path]:
        """Determine destination paths based on client code and job category"""
        client_code = json_data.get("ClientCode", "").upper()
        job_category = json_data.get("JobCategory", "").upper()
        destinations = []
        
        # Check for specific client routing by ClientCode
        if client_code in Config.DESTINATIONS:
            destinations.append(Config.DESTINATIONS[client_code])
        
        # Check for Celtra by JobCategory (only for eligible clients)
        if job_category == "CELTRA" and client_folder.upper() in Config.CELTRA_ELIGIBLE_CLIENTS:
            celtra_dest = Config.DESTINATIONS["CELTRA"]
            if celtra_dest not in destinations:
                destinations.append(celtra_dest)
        
        return destinations
    
    def store_file(self, file_path: Path, folder_name: str) -> bool:
        """Store file to JSON_STORE with 777 permissions"""
        try:
            store_dir = Config.JSON_STORE / folder_name
            store_dir.mkdir(parents=True, exist_ok=True)
            store_dir.chmod(0o777)
            
            destination = store_dir / file_path.name
            shutil.copy2(file_path, destination)
            destination.chmod(0o777)
            
            return True
            
        except Exception as e:
            self.logger.error(f"Failed to store {file_path.name}: {e}")
            return False
    
    def route_file(self, file_path: Path, destination: Path) -> bool:
        """Route file to specific destination with 777 permissions"""
        try:
            destination.mkdir(parents=True, exist_ok=True)
            destination.chmod(0o777)
            
            dest_file = destination / file_path.name
            shutil.copy2(file_path, dest_file)
            dest_file.chmod(0o777)
            
            return True
            
        except Exception as e:
            self.logger.error(f"Failed to route {file_path.name} to {destination}: {e}")
            return False
    
    def process_single_file(self, file_path: Path, is_startup: bool = False, is_periodic: bool = False) -> Dict[str, Any]:
        """Process a single JSON file with timing"""
        start_time = time.time()
        thread_name = threading.current_thread().name
        
        # Create tag for logging
        tags = []
        if is_startup:
            tags.append("STARTUP")
        if is_periodic:
            tags.append("PERIODIC")
        tag_str = f"[{'/'.join(tags)}] " if tags else ""
        
        try:
            # Check if already processed recently
            if not is_startup and self.was_recently_processed(file_path):
                self.logger.debug(f"[{thread_name}] {tag_str}Skipping recently processed: {file_path.name}")
                return {'success': True, 'skipped': True}
            
            self.logger.info(f"[{thread_name}] {tag_str}Processing: {file_path.name}")
            
            if not is_startup and not is_periodic:
                time.sleep(Config.WAIT_DELAY)
            
            # Parse JSON
            json_data = self.parse_json_file(file_path)
            if json_data is None:
                error_msg = "JSON parsing failed"
                folder_name = file_path.parent.name if file_path.parent != Config.HOT_FOLDER else "DEFAULT"
                processing_time = time.time() - start_time
                
                # Move failed file
                self.move_failed_file(file_path, error_msg)
                
                self.daily_stats.record_file_processed(
                    folder_name, {}, [], processing_time, False, error_msg, is_startup, is_periodic
                )
                return {'success': False, 'error': error_msg}
            
            # Determine folder name
            folder_name = file_path.parent.name if file_path.parent != Config.HOT_FOLDER else "DEFAULT"
            
            # Store file
            stored = self.store_file(file_path, folder_name)
            
            # Route file
            destinations = self.determine_routing(json_data, folder_name)
            routed_count = 0
            
            for destination in destinations:
                if self.route_file(file_path, destination):
                    routed_count += 1
            
            success = stored and (routed_count == len(destinations) if destinations else True)
            processing_time = time.time() - start_time
            
            # Record statistics
            self.daily_stats.record_file_processed(
                folder_name, json_data, destinations, processing_time, 
                success, None if success else "Processing failed", is_startup, is_periodic
            )
            
            if success:
                # Mark as processed and delete original
                self.add_to_recently_processed(file_path)
                file_path.unlink()
                
                route_info = f" + {len(destinations)} routes" if destinations else ""
                self.logger.info(f"[{thread_name}] {tag_str}✅ {file_path.name}{route_info} ({processing_time:.2f}s)")
            else:
                # Move failed file
                self.move_failed_file(file_path, "Processing failed")
            
            return {'success': success}
            
        except Exception as e:
            processing_time = time.time() - start_time
            folder_name = file_path.parent.name if file_path.parent != Config.HOT_FOLDER else "DEFAULT"
            
            # Move failed file
            self.move_failed_file(file_path, str(e))
            
            self.daily_stats.record_file_processed(
                folder_name, {}, [], processing_time, False, str(e), is_startup, is_periodic
            )
            
            self.logger.error(f"[{thread_name}] {tag_str}Error processing {file_path.name}: {e}")
            return {'success': False, 'error': str(e)}
    
    def process_batch(self, file_paths: List[Path], is_startup: bool = False, is_periodic: bool = False):
        """Process a batch of files concurrently"""
        if not file_paths:
            return
        
        batch_start = time.time()
        tags = []
        if is_startup:
            tags.append("STARTUP")
        if is_periodic:
            tags.append("PERIODIC")
        tag_str = f"[{'/'.join(tags)}] " if tags else ""
        
        self.logger.info(f"{tag_str}Processing batch of {len(file_paths)} files")
        
        with ThreadPoolExecutor(max_workers=min(Config.MAX_WORKERS, len(file_paths))) as executor:
            future_to_file = {
                executor.submit(self.process_single_file, file_path, is_startup, is_periodic): file_path 
                for file_path in file_paths
            }
            
            for future in as_completed(future_to_file):
                try:
                    future.result()
                except Exception as e:
                    file_path = future_to_file[future]
                    self.logger.error(f"{tag_str}Batch processing error for {file_path.name}: {e}")
        
        batch_time = time.time() - batch_start
        self.daily_stats.record_batch(len(file_paths))
        self.logger.info(f"{tag_str}Batch completed: {len(file_paths)} files in {batch_time:.2f}s")
    
    def queue_file(self, file_path: Path, is_periodic: bool = False):
        """Add file to processing queue"""
        if self.startup_complete:
            # Add flag to indicate if this is from periodic scan
            self.file_queue.put((file_path, is_periodic))
    
    def batch_processor_worker(self):
        """Background worker for batch processing"""
        while True:
            files_to_process = []
            periodic_flags = []
            batch_start_time = time.time()
            
            while (len(files_to_process) < Config.BATCH_SIZE and 
                   (time.time() - batch_start_time) < Config.BATCH_TIMEOUT):
                
                try:
                    file_data = self.file_queue.get(timeout=1.0)
                    if isinstance(file_data, tuple):
                        file_path, is_periodic = file_data
                    else:
                        file_path, is_periodic = file_data, False
                    
                    files_to_process.append(file_path)
                    periodic_flags.append(is_periodic)
                    self.file_queue.task_done()
                except:
                    if files_to_process:
                        break
                    continue
            
            if files_to_process and self.startup_complete:
                # Process batch - mark as periodic if any files are from periodic scan
                is_periodic_batch = any(periodic_flags)
                self.process_batch(files_to_process, is_startup=False, is_periodic=is_periodic_batch)
    
    def generate_daily_report(self):
        """Generate comprehensive daily report"""
        try:
            stats = self.daily_stats.get_stats_summary()
            
            # Generate report content
            report_content = self.format_daily_report(stats)
            
            # Save to file
            report_file = Config.REPORTS_DIR / f"daily_report_{stats['date']}.txt"
            with open(report_file, 'w') as f:
                f.write(report_content)
            
            # Email report
            self.email_daily_report(report_content, stats['date'])
            
            # Clean old reports
            self.cleanup_old_reports()
            
            # Reset stats for new day
            self.daily_stats.reset_stats()
            
            self.logger.info(f"Daily report generated: {report_file}")
            
        except Exception as e:
            self.logger.error(f"Failed to generate daily report: {e}")
    
    def format_daily_report(self, stats: Dict[str, Any]) -> str:
        """Format daily statistics into readable report"""
        startup_info = f"Startup Files Processed: {stats['startup_files_processed']}\n" if stats['startup_files_processed'] > 0 else ""
        periodic_info = f"Periodic Scan Files Found: {stats['periodic_files_found']}\n" if stats['periodic_files_found'] > 0 else ""
        failed_info = f"Failed Files Moved: {stats['failed_files_moved']}\n" if stats['failed_files_moved'] > 0 else ""
        
        report = f"""
JSON WORKFLOW DAILY REPORT - HYBRID VERSION
Date: {stats['date']}
Uptime: {stats['uptime']}

=== SUMMARY ===
Total Files Processed: {stats['total_processed']}
{startup_info}{periodic_info}{failed_info}Total Errors: {stats['total_errors']}
Success Rate: {stats['success_rate']:.1f}%
Total Batches: {stats['total_batches']}
Average Processing Time: {stats['avg_processing_time']}s
Average Batch Size: {stats['avg_batch_size']} files

=== CLIENT BREAKDOWN ===
"""
        
        for client, client_data in sorted(stats['client_stats'].items()):
            report += f"\n{client}:\n"
            report += f"  Processed: {client_data['processed']}\n"
            report += f"  Routed: {client_data['routed']}\n"
            report += f"  Celtra Projects: {client_data['celtra']}\n"
            report += f"  Errors: {client_data['errors']}\n"
            
            if client_data['job_categories']:
                report += f"  Job Categories: {dict(client_data['job_categories'])}\n"
        
        report += "\n=== DESTINATION BREAKDOWN ===\n"
        for dest, count in sorted(stats['destination_stats'].items()):
            report += f"{dest}: {count} files\n"
        
        report += "\n=== HOURLY BREAKDOWN ===\n"
        for hour in range(24):
            count = stats['hourly_stats'].get(hour, 0)
            if count > 0:
                report += f"{hour:02d}:00 - {count} files\n"
        
        if stats['error_details']:
            report += "\n=== ERROR DETAILS ===\n"
            for error in stats['error_details'][-10:]:
                report += f"{error['time']} - {error['client']}: {error['error']}\n"
        
        return report
    
    def email_daily_report(self, report_content: str, report_date: str):
        """Email the daily report"""
        try:
            msg = MIMEMultipart()
            msg['From'] = Config.SENDER_EMAIL
            msg['To'] = ', '.join(Config.REPORT_EMAILS)
            msg['Subject'] = f"JSON Workflow Daily Report - {report_date}"
            
            msg.attach(MIMEText(report_content, 'plain'))
            
            server = smtplib.SMTP(Config.SMTP_SERVER, Config.SMTP_PORT)
            server.starttls()
            server.login(Config.SMTP_USER, Config.SMTP_PASSWORD)
            server.sendmail(Config.SENDER_EMAIL, Config.REPORT_EMAILS, msg.as_string())
            server.quit()
            
            self.logger.info("Daily report emailed successfully")
            
        except Exception as e:
            self.logger.error(f"Failed to email daily report: {e}")
    
    def cleanup_old_reports(self):
        """Remove old report files"""
        try:
            cutoff_date = date.today() - timedelta(days=Config.KEEP_REPORTS_DAYS)
            
            for report_file in Config.REPORTS_DIR.glob("daily_report_*.txt"):
                try:
                    file_date_str = report_file.stem.split('_')[-1]
                    file_date = datetime.strptime(file_date_str, '%Y-%m-%d').date()
                    
                    if file_date < cutoff_date:
                        report_file.unlink()
                        self.logger.debug(f"Deleted old report: {report_file}")
                        
                except Exception:
                    pass
                    
        except Exception as e:
            self.logger.error(f"Failed to cleanup old reports: {e}")
    
    def get_current_stats(self) -> str:
        """Get current statistics as formatted string"""
        stats = self.daily_stats.get_stats_summary()
        startup_info = f" (startup: {stats['startup_files_processed']})" if stats['startup_files_processed'] > 0 else ""
        periodic_info = f" (periodic: {stats['periodic_files_found']})" if stats['periodic_files_found'] > 0 else ""
        failed_info = f" (failed: {stats['failed_files_moved']})" if stats['failed_files_moved'] > 0 else ""
        return f"Today: {stats['total_processed']} processed{startup_info}{periodic_info}{failed_info}, {stats['total_errors']} errors, {stats['success_rate']:.1f}% success"


class HybridFileHandler(FileSystemEventHandler):
    """File system event handler for hybrid monitoring"""
    
    def __init__(self, processor: HybridProcessor):
        self.processor = processor
        self.logger = logging.getLogger(__name__)
    
    def on_created(self, event):
        if event.is_directory or not self.processor.startup_complete:
            return
        
        file_path = Path(event.src_path)
        if file_path.suffix.lower() == '.json':
            self.logger.debug(f"📥 New file detected: {file_path.name}")
            self.processor.queue_file(file_path, is_periodic=False)
    
    def on_moved(self, event):
        if event.is_directory or not self.processor.startup_complete:
            return
        
        dest_path = Path(event.dest_path)
        if dest_path.suffix.lower() == '.json':
            self.logger.debug(f"📁 File moved to hot folder: {dest_path.name}")
            self.processor.queue_file(dest_path, is_periodic=False)


def main():
    """Main entry point with hybrid monitoring"""
    processor = HybridProcessor()
    
    # Step 1: Scan and process existing files
    existing_files = processor.scan_existing_files()
    processor.process_startup_files(existing_files)
    
    # Step 2: Mark startup complete and begin real-time monitoring
    processor.startup_complete = True
    processor.logger.info("🎯 Startup scan complete - switching to hybrid monitoring")
    
    # Start batch processing worker for real-time files
    batch_worker = threading.Thread(
        target=processor.batch_processor_worker,
        name="BatchWorker",
        daemon=True
    )
    batch_worker.start()
    
    # Start periodic scanning thread
    periodic_scanner = threading.Thread(
        target=processor.periodic_scan,
        name="PeriodicScanner",
        daemon=True
    )
    periodic_scanner.start()
    
    # Setup file monitoring for new files
    event_handler = HybridFileHandler(processor)
    observer = Observer()
    observer.schedule(event_handler, str(Config.HOT_FOLDER), recursive=True)
    
    # Start real-time monitoring
    observer.start()
    processor.logger.info(f"🔄 Hybrid monitoring active: Events + {Config.PERIODIC_SCAN_INTERVAL}s periodic scans")
    
    try:
        while True:
            time.sleep(60)
            current_stats = processor.get_current_stats()
            processor.logger.info(f"Stats: {current_stats}")
    except KeyboardInterrupt:
        processor.logger.info("Stopping Hybrid JSON Workflow Processor...")
        observer.stop()
    
    observer.join()
    processor.logger.info("Hybrid JSON Workflow Processor stopped")


if __name__ == "__main__":
    main()