Add daily summary report with comprehensive statistics
Automated daily report that analyzes all workflow logs and sends email summary at 7pm. NEW SCRIPT: daily_report.py - Analyzes last 24 hours of all workflow logs - Parses campaign and asset statistics - Extracts error messages - Sends comprehensive email report - Run via cron at 7pm daily FEATURES: ✓ Overall summary statistics across all workflows ✓ Per-workflow breakdown (A1→A2, A2→A3, A5→A6, B1→B2) ✓ Success rate calculation ✓ Campaign details with status (completed/partial/no_assets) ✓ Error log extraction ✓ Collapsible sections for campaign and error details ✓ Color-coded metrics (green=good, orange=warning, red=error) STATISTICS TRACKED: - Campaigns found/processed/completed/partial - Total assets processed/successful/failed - NOT APPROVED assets (A5→A6) - Skipped assets (approved items) - Success rate percentage - Error count and messages EMAIL TEMPLATE: daily_report - Blue theme (#1976d2) - Grid layout for statistics - Collapsible campaign details - Collapsible error logs (max 10 shown) - Professional dashboard-style formatting USAGE: python scripts/daily_report.py CRON SCHEDULE (7pm daily): 0 19 * * * cd ~/Python-Version && venv/bin/python scripts/daily_report.py >> logs/daily_report.log 2>&1 TESTED: ✓ Successfully parsed all 4 workflow logs ✓ Found 26 campaigns, 58 assets from last 24 hours ✓ Email sent successfully with all statistics ✓ Success rate: 48.3% (from test data) ✓ Error extraction working Changes: - NEW: scripts/daily_report.py (241 lines) - EDIT: shared/notifier.py (add daily_report template) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
DJP
parent
1081cfd9ca
commit
c720759cc7
2 changed files with 376 additions and 0 deletions
273
Python-Version/scripts/daily_report.py
Executable file
273
Python-Version/scripts/daily_report.py
Executable file
|
|
@ -0,0 +1,273 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Daily Report Generator
|
||||
Analyzes log files from all workflows and sends comprehensive daily summary email
|
||||
Run at 7pm daily via cron
|
||||
Compatible with Python 3.6+
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
import logging
|
||||
|
||||
# Add shared library to path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
|
||||
from shared.config_loader import load_config
|
||||
from shared.notifier import Notifier
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
|
||||
logger = logging.getLogger('DailyReport')
|
||||
|
||||
def parse_log_file(log_path, since_hours=24):
|
||||
"""
|
||||
Parse log file and extract statistics for the last N hours
|
||||
|
||||
Args:
|
||||
log_path: Path to log file
|
||||
since_hours: How many hours back to analyze (default 24)
|
||||
|
||||
Returns:
|
||||
dict with statistics
|
||||
"""
|
||||
stats = {
|
||||
'campaigns_found': 0,
|
||||
'campaigns_processed': 0,
|
||||
'campaigns_completed': 0,
|
||||
'campaigns_partial': 0,
|
||||
'campaigns_no_assets': 0,
|
||||
'total_assets': 0,
|
||||
'assets_successful': 0,
|
||||
'assets_failed': 0,
|
||||
'assets_skipped': 0,
|
||||
'not_approved_count': 0,
|
||||
'errors': [],
|
||||
'campaign_details': []
|
||||
}
|
||||
|
||||
if not os.path.exists(log_path):
|
||||
logger.warning("Log file not found: {}".format(log_path))
|
||||
return stats
|
||||
|
||||
# Calculate cutoff time
|
||||
cutoff_time = datetime.now() - timedelta(hours=since_hours)
|
||||
|
||||
current_campaign = None
|
||||
|
||||
try:
|
||||
with open(log_path, 'r') as f:
|
||||
for line in f:
|
||||
# Parse timestamp
|
||||
match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line)
|
||||
if match:
|
||||
log_time = datetime.strptime(match.group(1), '%Y-%m-%d %H:%M:%S')
|
||||
if log_time < cutoff_time:
|
||||
continue
|
||||
|
||||
# Campaign found
|
||||
if 'Found' in line and 'campaigns' in line and 'status' in line:
|
||||
match = re.search(r'Found (\d+) campaigns with status', line)
|
||||
if match:
|
||||
stats['campaigns_found'] += int(match.group(1))
|
||||
|
||||
# Processing campaign
|
||||
if 'Processing campaign:' in line:
|
||||
match = re.search(r'Processing campaign: ([^(]+)\(([^)]+)\)', line)
|
||||
if match:
|
||||
current_campaign = {
|
||||
'name': match.group(1).strip(),
|
||||
'number': match.group(2).strip(),
|
||||
'total_assets': 0,
|
||||
'successful': 0,
|
||||
'failed': 0,
|
||||
'skipped': 0,
|
||||
'status': 'unknown'
|
||||
}
|
||||
stats['campaigns_processed'] += 1
|
||||
|
||||
# Total assets found
|
||||
if current_campaign:
|
||||
if 'Found' in line and 'master assets' in line:
|
||||
match = re.search(r'Found (\d+) master assets', line)
|
||||
if match:
|
||||
current_campaign['total_assets'] = int(match.group(1))
|
||||
stats['total_assets'] += int(match.group(1))
|
||||
|
||||
# NOT APPROVED count (A5→A6 only)
|
||||
if 'NOT APPROVED (rejected) assets:' in line:
|
||||
match = re.search(r'NOT APPROVED \(rejected\) assets: (\d+)', line)
|
||||
if match:
|
||||
count = int(match.group(1))
|
||||
stats['not_approved_count'] += count
|
||||
current_campaign['not_approved'] = count
|
||||
|
||||
# Skipped count (A5→A6 only)
|
||||
if 'Approved/other status (skipped):' in line:
|
||||
match = re.search(r'Approved/other status \(skipped\): (\d+)', line)
|
||||
if match:
|
||||
skipped = int(match.group(1))
|
||||
stats['assets_skipped'] += skipped
|
||||
current_campaign['skipped'] = skipped
|
||||
|
||||
# Success count
|
||||
if 'Successfully processed:' in line or 'Successful:' in line:
|
||||
match = re.search(r'(Successfully processed|Successful): (\d+)', line)
|
||||
if match:
|
||||
count = int(match.group(2))
|
||||
current_campaign['successful'] = count
|
||||
stats['assets_successful'] += count
|
||||
|
||||
# Failed count
|
||||
if 'Failed:' in line and 'assets' in line:
|
||||
match = re.search(r'Failed: (\d+)', line)
|
||||
if match:
|
||||
count = int(match.group(1))
|
||||
current_campaign['failed'] = count
|
||||
stats['assets_failed'] += count
|
||||
|
||||
# Status updated
|
||||
if 'Status updated successfully' in line or 'Campaign completed successfully' in line:
|
||||
current_campaign['status'] = 'completed'
|
||||
stats['campaigns_completed'] += 1
|
||||
|
||||
# Partial completion
|
||||
if 'Campaign incomplete' in line or 'Status NOT updated' in line:
|
||||
if current_campaign['status'] != 'completed':
|
||||
current_campaign['status'] = 'partial'
|
||||
stats['campaigns_partial'] += 1
|
||||
|
||||
# No assets found
|
||||
if 'No master assets found' in line or 'No NOT APPROVED assets found' in line:
|
||||
current_campaign['status'] = 'no_assets'
|
||||
stats['campaigns_no_assets'] += 1
|
||||
|
||||
# Errors
|
||||
if ' - ERROR - ' in line or ' - CRITICAL - ' in line:
|
||||
error_msg = line.split(' - ')[-1].strip()
|
||||
if error_msg not in stats['errors']:
|
||||
stats['errors'].append(error_msg)
|
||||
|
||||
# Add last campaign if exists
|
||||
if current_campaign and current_campaign not in stats['campaign_details']:
|
||||
stats['campaign_details'].append(current_campaign)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error parsing log {}: {}".format(log_path, str(e)))
|
||||
|
||||
return stats
|
||||
|
||||
def generate_daily_report():
|
||||
"""Generate daily report for all workflows"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Generating Daily Report")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Load configuration
|
||||
config = load_config('config/config.yaml')
|
||||
notifier = Notifier(config)
|
||||
|
||||
# Log files to analyze
|
||||
log_files = {
|
||||
'A1→A2 (Master Assets)': 'logs/a1_to_a2.log',
|
||||
'A2→A3 (Upload from Box)': 'logs/a2_to_a3.log',
|
||||
'A5→A6 (Rework/Rejections)': 'logs/a5_to_a6.log',
|
||||
'B1→B2 (Global Masters)': 'logs/b1_to_b2.log'
|
||||
}
|
||||
|
||||
# Collect stats for each workflow
|
||||
workflow_stats = {}
|
||||
total_stats = {
|
||||
'campaigns_found': 0,
|
||||
'campaigns_processed': 0,
|
||||
'campaigns_completed': 0,
|
||||
'campaigns_partial': 0,
|
||||
'campaigns_no_assets': 0,
|
||||
'total_assets': 0,
|
||||
'assets_successful': 0,
|
||||
'assets_failed': 0,
|
||||
'assets_skipped': 0,
|
||||
'not_approved_count': 0,
|
||||
'total_errors': 0
|
||||
}
|
||||
|
||||
for workflow_name, log_path in log_files.items():
|
||||
logger.info("Analyzing: {}".format(workflow_name))
|
||||
stats = parse_log_file(log_path, since_hours=24)
|
||||
workflow_stats[workflow_name] = stats
|
||||
|
||||
# Aggregate totals
|
||||
total_stats['campaigns_found'] += stats['campaigns_found']
|
||||
total_stats['campaigns_processed'] += stats['campaigns_processed']
|
||||
total_stats['campaigns_completed'] += stats['campaigns_completed']
|
||||
total_stats['campaigns_partial'] += stats['campaigns_partial']
|
||||
total_stats['campaigns_no_assets'] += stats['campaigns_no_assets']
|
||||
total_stats['total_assets'] += stats['total_assets']
|
||||
total_stats['assets_successful'] += stats['assets_successful']
|
||||
total_stats['assets_failed'] += stats['assets_failed']
|
||||
total_stats['assets_skipped'] += stats['assets_skipped']
|
||||
total_stats['not_approved_count'] += stats['not_approved_count']
|
||||
total_stats['total_errors'] += len(stats['errors'])
|
||||
|
||||
# Calculate success rate
|
||||
if total_stats['total_assets'] > 0:
|
||||
success_rate = (total_stats['assets_successful'] / total_stats['total_assets']) * 100
|
||||
else:
|
||||
success_rate = 0
|
||||
|
||||
total_stats['success_rate'] = success_rate
|
||||
|
||||
# Generate report timestamp
|
||||
report_date = datetime.now().strftime('%Y-%m-%d')
|
||||
report_time = datetime.now().strftime('%I:%M %p')
|
||||
|
||||
# Send email report
|
||||
logger.info("")
|
||||
logger.info("Sending daily report email...")
|
||||
|
||||
notifier.send_email(
|
||||
template_name='daily_report',
|
||||
recipients=config['notifications']['recipients']['success'],
|
||||
data={
|
||||
'report_date': report_date,
|
||||
'report_time': report_time,
|
||||
'total_stats': total_stats,
|
||||
'workflow_stats': workflow_stats
|
||||
}
|
||||
)
|
||||
|
||||
logger.info("✓ Daily report sent successfully")
|
||||
logger.info("")
|
||||
logger.info("=" * 60)
|
||||
logger.info("DAILY REPORT SUMMARY")
|
||||
logger.info("=" * 60)
|
||||
logger.info("Campaigns Found: {}".format(total_stats['campaigns_found']))
|
||||
logger.info("Campaigns Processed: {}".format(total_stats['campaigns_processed']))
|
||||
logger.info(" - Completed: {}".format(total_stats['campaigns_completed']))
|
||||
logger.info(" - Partial: {}".format(total_stats['campaigns_partial']))
|
||||
logger.info(" - No Assets: {}".format(total_stats['campaigns_no_assets']))
|
||||
logger.info("Total Assets: {}".format(total_stats['total_assets']))
|
||||
logger.info(" - Successful: {}".format(total_stats['assets_successful']))
|
||||
logger.info(" - Failed: {}".format(total_stats['assets_failed']))
|
||||
logger.info(" - Skipped: {}".format(total_stats['assets_skipped']))
|
||||
if total_stats['not_approved_count'] > 0:
|
||||
logger.info(" - NOT APPROVED: {}".format(total_stats['not_approved_count']))
|
||||
logger.info("Success Rate: {:.1f}%".format(success_rate))
|
||||
logger.info("Total Errors: {}".format(total_stats['total_errors']))
|
||||
logger.info("=" * 60)
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
generate_daily_report()
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
logger.error("Daily report failed: {}".format(str(e)))
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
|
@ -540,6 +540,109 @@ class Notifier:
|
|||
<p style="color: #666; font-size: 12px; margin-top: 20px;">B1→B2 script completed with no assets to process.</p>
|
||||
</div>
|
||||
"""
|
||||
},
|
||||
'daily_report': {
|
||||
'subject': "📊 Ferrero Automation Daily Report - {report_date}",
|
||||
'html': """
|
||||
<div style="font-family: Arial, sans-serif; max-width: 1000px; margin: 0 auto;">
|
||||
<div style="background-color: #1976d2; color: white; padding: 20px; text-align: center; border-radius: 8px 8px 0 0;">
|
||||
<h1 style="margin: 0;">📊 Ferrero Automation Daily Report</h1>
|
||||
<p style="margin: 10px 0 0 0; opacity: 0.9;">{{ report_date }} - Generated at {{ report_time }}</p>
|
||||
</div>
|
||||
|
||||
<!-- Overall Summary -->
|
||||
<div style="background-color: #e3f2fd; border-left: 4px solid #1976d2; padding: 15px; margin: 20px 0;">
|
||||
<h2 style="margin: 0 0 10px 0; color: #1976d2;">📈 Overall Summary (Last 24 Hours)</h2>
|
||||
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px; margin-top: 10px;">
|
||||
<div><strong>Campaigns Found:</strong> {{ total_stats.campaigns_found }}</div>
|
||||
<div><strong>Campaigns Processed:</strong> {{ total_stats.campaigns_processed }}</div>
|
||||
<div><strong>✅ Completed:</strong> <span style="color: #28a745;">{{ total_stats.campaigns_completed }}</span></div>
|
||||
<div><strong>⚠️ Partial:</strong> <span style="color: #ff9800;">{{ total_stats.campaigns_partial }}</span></div>
|
||||
<div><strong>No Assets:</strong> {{ total_stats.campaigns_no_assets }}</div>
|
||||
<div><strong>Total Assets:</strong> {{ total_stats.total_assets }}</div>
|
||||
<div><strong>✓ Successful:</strong> <span style="color: #28a745;">{{ total_stats.assets_successful }}</span></div>
|
||||
<div><strong>✗ Failed:</strong> <span style="color: #d32f2f;">{{ total_stats.assets_failed }}</span></div>
|
||||
{% if total_stats.assets_skipped > 0 %}<div><strong>Skipped (Approved):</strong> {{ total_stats.assets_skipped }}</div>{% endif %}
|
||||
{% if total_stats.not_approved_count > 0 %}<div><strong>🔴 NOT APPROVED:</strong> {{ total_stats.not_approved_count }}</div>{% endif %}
|
||||
</div>
|
||||
{% if total_stats.total_assets > 0 %}
|
||||
<div style="margin-top: 15px; padding: 10px; background-color: white; border-radius: 4px;">
|
||||
<strong>Success Rate:</strong>
|
||||
<span style="font-size: 24px; font-weight: bold; color: {% if total_stats.success_rate >= 95 %}#28a745{% elif total_stats.success_rate >= 80 %}#ff9800{% else %}#d32f2f{% endif %};">
|
||||
{{ "%.1f"|format(total_stats.success_rate) }}%
|
||||
</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Workflow Breakdown -->
|
||||
<h2 style="margin: 30px 0 20px 0; color: #333;">🔄 Workflow Breakdown</h2>
|
||||
|
||||
{% for workflow_name, stats in workflow_stats.items() %}
|
||||
<div style="border: 1px solid #ddd; margin: 15px 0; padding: 0; background-color: #fafafa; border-radius: 4px;">
|
||||
<div style="background-color: #424242; color: white; padding: 12px 15px; border-radius: 4px 4px 0 0;">
|
||||
<strong>{{ workflow_name }}</strong>
|
||||
</div>
|
||||
<div style="padding: 15px;">
|
||||
{% if stats.campaigns_processed > 0 %}
|
||||
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 8px; margin-bottom: 10px;">
|
||||
<div><strong>Campaigns:</strong> {{ stats.campaigns_processed }}</div>
|
||||
<div><strong>Assets:</strong> {{ stats.total_assets }}</div>
|
||||
<div style="color: #28a745;"><strong>✓ Successful:</strong> {{ stats.assets_successful }}</div>
|
||||
<div style="color: #d32f2f;"><strong>✗ Failed:</strong> {{ stats.assets_failed }}</div>
|
||||
{% if stats.assets_skipped > 0 %}<div><strong>Skipped:</strong> {{ stats.assets_skipped }}</div>{% endif %}
|
||||
{% if stats.not_approved_count > 0 %}<div style="color: #d32f2f;"><strong>NOT APPROVED:</strong> {{ stats.not_approved_count }}</div>{% endif %}
|
||||
</div>
|
||||
|
||||
{% if stats.campaign_details %}
|
||||
<details style="margin-top: 10px;">
|
||||
<summary style="cursor: pointer; font-weight: bold; color: #1976d2;">View Campaign Details ({{ stats.campaign_details|length }})</summary>
|
||||
<div style="margin-top: 10px;">
|
||||
{% for campaign in stats.campaign_details %}
|
||||
<div style="padding: 8px; margin: 5px 0; background-color: white; border-left: 3px solid {% if campaign.status == 'completed' %}#28a745{% elif campaign.status == 'partial' %}#ff9800{% else %}#999{% endif %}; border-radius: 2px;">
|
||||
<strong>{{ campaign.name }}</strong> ({{ campaign.number }})
|
||||
<br><small>Total: {{ campaign.total_assets }}, Success: {{ campaign.successful }}, Failed: {{ campaign.failed }}{% if campaign.get('skipped') %}, Skipped: {{ campaign.skipped }}{% endif %}</small>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</details>
|
||||
{% endif %}
|
||||
|
||||
{% if stats.errors %}
|
||||
<details style="margin-top: 10px;">
|
||||
<summary style="cursor: pointer; font-weight: bold; color: #d32f2f;">⚠️ Errors ({{ stats.errors|length }})</summary>
|
||||
<div style="margin-top: 10px;">
|
||||
{% for error in stats.errors[:10] %}
|
||||
<div style="padding: 5px; margin: 3px 0; background-color: #ffebee; border-radius: 2px; font-size: 12px;">
|
||||
{{ error }}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% if stats.errors|length > 10 %}
|
||||
<div style="padding: 5px; margin: 3px 0; font-style: italic; color: #666;">
|
||||
... and {{ stats.errors|length - 10 }} more errors
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<p style="color: #999; font-style: italic; margin: 0;">No activity in the last 24 hours</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
<!-- Footer -->
|
||||
<div style="background-color: #f8f9fa; padding: 15px; margin: 20px 0; border-radius: 4px; text-align: center;">
|
||||
<p style="margin: 0; color: #666;">
|
||||
<strong>📌 Automation Scripts:</strong> A1→A2, A2→A3, A5→A6, B1→B2
|
||||
</p>
|
||||
<p style="margin: 5px 0 0 0; color: #666; font-size: 12px;">
|
||||
Scripts run every 5 minutes | Report generated daily at 7:00 PM
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue