#!/usr/bin/env python3 """ Daily Report Generator Analyzes log files from all workflows and sends comprehensive daily summary email Run at 7pm daily via cron Compatible with Python 3.6+ """ import sys import os import re from datetime import datetime, timedelta import logging # Add shared library to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from shared.config_loader import load_config from shared.notifier import Notifier # Setup logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger('DailyReport') def parse_log_file(log_path, since_hours=24): """ Parse log file and extract statistics for the last N hours Args: log_path: Path to log file since_hours: How many hours back to analyze (default 24) Returns: dict with statistics """ stats = { 'campaigns_found': 0, 'campaigns_processed': 0, 'campaigns_completed': 0, 'campaigns_partial': 0, 'campaigns_no_assets': 0, 'total_assets': 0, 'assets_successful': 0, 'assets_failed': 0, 'assets_skipped': 0, 'not_approved_count': 0, 'errors': [], 'campaign_details': [] } if not os.path.exists(log_path): logger.warning("Log file not found: {}".format(log_path)) return stats # Calculate cutoff time cutoff_time = datetime.now() - timedelta(hours=since_hours) current_campaign = None in_summary_block = False # Track when we're inside a summary block try: with open(log_path, 'r') as f: for line in f: # Parse timestamp match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line) if match: log_time = datetime.strptime(match.group(1), '%Y-%m-%d %H:%M:%S') # Skip old lines UNLESS we're in a summary block (need to parse complete summary) if log_time < cutoff_time and not in_summary_block: continue # Campaign found if 'Found' in line and 'campaigns' in line and 'status' in line: match = re.search(r'Found (\d+) campaigns with status', line) if match: stats['campaigns_found'] += int(match.group(1)) # Processing campaign if 'Processing campaign:' in line: match = re.search(r'Processing campaign: ([^(]+)\(([^)]+)\)', line) if match: current_campaign = { 'name': match.group(1).strip(), 'number': match.group(2).strip(), 'total_assets': 0, 'successful': 0, 'failed': 0, 'skipped': 0, 'status': 'unknown' } stats['campaigns_processed'] += 1 # Total assets found if current_campaign: if 'Found' in line and 'master assets' in line: match = re.search(r'Found (\d+) master assets', line) if match: current_campaign['total_assets'] = int(match.group(1)) stats['total_assets'] += int(match.group(1)) # NOT APPROVED count (A5→A6 only) if 'NOT APPROVED (rejected) assets:' in line: match = re.search(r'NOT APPROVED \(rejected\) assets: (\d+)', line) if match: count = int(match.group(1)) stats['not_approved_count'] += count current_campaign['not_approved'] = count # Skipped count (A5→A6 only) if 'Approved/other status (skipped):' in line: match = re.search(r'Approved/other status \(skipped\): (\d+)', line) if match: skipped = int(match.group(1)) stats['assets_skipped'] += skipped current_campaign['skipped'] = skipped # Success count if 'Successfully processed:' in line or 'Successful:' in line: match = re.search(r'(Successfully processed|Successful): (\d+)', line) if match: count = int(match.group(2)) current_campaign['successful'] = count stats['assets_successful'] += count # Failed count if 'Failed:' in line and 'assets' in line: match = re.search(r'Failed: (\d+)', line) if match: count = int(match.group(1)) current_campaign['failed'] = count stats['assets_failed'] += count # Status updated if 'Status updated successfully' in line or 'Campaign completed successfully' in line: current_campaign['status'] = 'completed' stats['campaigns_completed'] += 1 # Partial completion if 'Campaign incomplete' in line or 'Status NOT updated' in line: if current_campaign['status'] != 'completed': current_campaign['status'] = 'partial' stats['campaigns_partial'] += 1 # No assets found if 'No master assets found' in line or 'No NOT APPROVED assets found' in line: current_campaign['status'] = 'no_assets' stats['campaigns_no_assets'] += 1 # Detect start of summary blocks if 'Processing Summary' in line: in_summary_block = True summary_started = True # Flag to skip the first separator # Extract timestamp from current line for batch identification timestamp_match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line) batch_time = timestamp_match.group(1) if timestamp_match else 'Unknown Time' # Initialize batch tracking for this summary current_batch = { 'name': 'Upload Batch', 'number': batch_time, # Use timestamp as identifier 'total_assets': 0, 'successful': 0, 'failed': 0, 'status': 'completed' } elif in_summary_block and '====' in line: # First separator after header - skip it if 'summary_started' in locals() and summary_started: summary_started = False else: # Second separator - end of block in_summary_block = False # Add batch to campaign details if it has data if current_batch and current_batch['total_assets'] > 0: stats['campaign_details'].append(current_batch) stats['campaigns_processed'] += 1 # Count this batch as a processed campaign current_batch = None # Parse summary block contents if in_summary_block: # Parse file/asset counts from summary blocks (handle leading whitespace) if 'Total files processed:' in line: match = re.search(r'Total files processed:\s*(\d+)', line) if match: count = int(match.group(1)) stats['total_assets'] += count if current_batch: current_batch['total_assets'] = count if 'Total campaigns processed:' in line: match = re.search(r'Total campaigns processed:\s*(\d+)', line) if match: count = int(match.group(1)) stats['campaigns_processed'] += count # Success/failure counts in summary blocks if 'Successful:' in line and 'Successfully processed' not in line: match = re.search(r'Successful:\s*(\d+)', line) if match: count = int(match.group(1)) stats['assets_successful'] += count if current_batch: current_batch['successful'] = count if 'Failed:' in line and 'Failed to' not in line: match = re.search(r'Failed:\s*(\d+)', line) if match: count = int(match.group(1)) stats['assets_failed'] += count if current_batch: current_batch['failed'] = count # Errors if ' - ERROR - ' in line or ' - CRITICAL - ' in line: error_msg = line.split(' - ')[-1].strip() if error_msg not in stats['errors']: stats['errors'].append(error_msg) # Add last campaign if exists if current_campaign and current_campaign not in stats['campaign_details']: stats['campaign_details'].append(current_campaign) except Exception as e: logger.error("Error parsing log {}: {}".format(log_path, str(e))) return stats def generate_daily_report(): """Generate daily report for all workflows""" logger.info("=" * 60) logger.info("Generating Daily Report") logger.info("=" * 60) # Load configuration config = load_config('config/config.yaml') notifier = Notifier(config) # Log files to analyze log_files = { 'A1→A2 (Master Assets)': 'logs/a1_to_a2.log', 'A1→A2 (Box Upload)': 'logs/a1_to_a2_box.log', 'A2→A3 (Upload from Box)': 'logs/a2_to_a3.log', 'A4 (Box Upload)': 'logs/a4_box.log', 'A4 (Webhook Monitor)': 'logs/a4_webhook.log', 'A5→A6 (Rework/Rejections)': 'logs/a5_to_a6.log', 'B1→B2 (Global Masters)': 'logs/b1_to_b2.log' } # Collect stats for each workflow workflow_stats = {} total_stats = { 'campaigns_found': 0, 'campaigns_processed': 0, 'campaigns_completed': 0, 'campaigns_partial': 0, 'campaigns_no_assets': 0, 'total_assets': 0, 'assets_successful': 0, 'assets_failed': 0, 'assets_skipped': 0, 'not_approved_count': 0, 'total_errors': 0 } for workflow_name, log_path in log_files.items(): logger.info("Analyzing: {}".format(workflow_name)) stats = parse_log_file(log_path, since_hours=24) workflow_stats[workflow_name] = stats # Aggregate totals total_stats['campaigns_found'] += stats['campaigns_found'] total_stats['campaigns_processed'] += stats['campaigns_processed'] total_stats['campaigns_completed'] += stats['campaigns_completed'] total_stats['campaigns_partial'] += stats['campaigns_partial'] total_stats['campaigns_no_assets'] += stats['campaigns_no_assets'] total_stats['total_assets'] += stats['total_assets'] total_stats['assets_successful'] += stats['assets_successful'] total_stats['assets_failed'] += stats['assets_failed'] total_stats['assets_skipped'] += stats['assets_skipped'] total_stats['not_approved_count'] += stats['not_approved_count'] total_stats['total_errors'] += len(stats['errors']) # Calculate success rate if total_stats['total_assets'] > 0: success_rate = (total_stats['assets_successful'] / total_stats['total_assets']) * 100 else: success_rate = 0 total_stats['success_rate'] = success_rate # Generate report timestamp report_date = datetime.now().strftime('%Y-%m-%d') report_time = datetime.now().strftime('%I:%M %p') # Send email report logger.info("") logger.info("Sending daily report email...") notifier.send_email( template_name='daily_report', recipients=config['notifications']['recipients']['success'], data={ 'report_date': report_date, 'report_time': report_time, 'total_stats': total_stats, 'workflow_stats': workflow_stats } ) logger.info("✓ Daily report sent successfully") logger.info("") logger.info("=" * 60) logger.info("DAILY REPORT SUMMARY") logger.info("=" * 60) logger.info("Campaigns Found: {}".format(total_stats['campaigns_found'])) logger.info("Campaigns Processed: {}".format(total_stats['campaigns_processed'])) logger.info(" - Completed: {}".format(total_stats['campaigns_completed'])) logger.info(" - Partial: {}".format(total_stats['campaigns_partial'])) logger.info(" - No Assets: {}".format(total_stats['campaigns_no_assets'])) logger.info("Total Assets: {}".format(total_stats['total_assets'])) logger.info(" - Successful: {}".format(total_stats['assets_successful'])) logger.info(" - Failed: {}".format(total_stats['assets_failed'])) logger.info(" - Skipped: {}".format(total_stats['assets_skipped'])) if total_stats['not_approved_count'] > 0: logger.info(" - NOT APPROVED: {}".format(total_stats['not_approved_count'])) logger.info("Success Rate: {:.1f}%".format(success_rate)) logger.info("Total Errors: {}".format(total_stats['total_errors'])) logger.info("=" * 60) if __name__ == '__main__': try: generate_daily_report() sys.exit(0) except Exception as e: logger.error("Daily report failed: {}".format(str(e))) import traceback traceback.print_exc() sys.exit(1)