fix: оптимизация алертинга Prometheus и Alertmanager

Исправлены критические проблемы с избыточными уведомлениями:

1. Alertmanager (config.yml):
   - group_wait: 10s → 30s (уменьшен спам повторных алертов)
   - group_interval: 10s → 5m (алерты группируются правильно)
   - repeat_interval: 1h → 4h (повторные уведомления раз в 4 часа)
   - Добавлена группировка по severity и instance
   - Исправлен шаблон Slack для отображения деталей алертов

2. Prometheus правила (alerts.yml):
   - ContainerHighMemory: порог 90% → 95%, for: 2m → 5m
   - WebsiteDown: for: 1m → 10m (синхронизировано со scrape_interval)
   - Добавлены детальные описания в alerts

Результат: количество уведомлений снижено с 90+ до минимума,
уведомления теперь содержат полную информацию о проблеме.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

opt/04-tools/monitoring/alertmanager/config.yml

@@ -2,10 +2,10 @@ global:
   slack_api_url: 'https://hooks.slack.com/services/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'
 
 route:
-  group_by: ['alertname']
-  group_wait: 10s
-  group_interval: 10s
-  repeat_interval: 1h
+  group_by: ['alertname', 'severity', 'instance']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
   receiver: 'slack-default'
 
 receivers:
@@ -13,15 +13,18 @@ receivers:
   slack_configs:
   - channel: '#server-status'
     send_resolved: true
-    title: '{{ if eq .Status "firing" }}🚨{{ else }}✅{{ end }} {{ .GroupLabels.alertname }}'
+    title: '{{ if eq .Status "firing" }}🚨{{ else }}✅{{ end }} {{ .GroupLabels.alertname }} ({{ .GroupLabels.severity }})'
     text: |-
       {{ if eq .Status "firing" }}*🔥 Problem Detected*{{ else }}*✅ Problem Resolved*{{ end }}
-      
-      *Summary:* {{ .CommonAnnotations.summary }}
-      *Details:* {{ .CommonAnnotations.description }}
+      {{ range .Alerts }}
+      *Alert:* {{ .Labels.alertname }}
+      *Summary:* {{ .Annotations.summary }}
+      *Details:* {{ .Annotations.description }}
+      *Instance:* {{ .Labels.instance }}{{ if .Labels.container }} ({{ .Labels.container }}){{ end }}
+      {{ end }}
       *Severity:* {{ .CommonLabels.severity }}
-      
+
       *Links:*
-      • <http://localhost:3000|Grafana>
-      • <http://localhost:9090|Prometheus>
+      • <https://grafana.ai-impress.com|Grafana>
+      • <https://prometheus.ai-impress.com|Prometheus>
     color: '{{ if eq .Status "firing" }}{{ if eq .CommonLabels.severity "critical" }}danger{{ else }}warning{{ end }}{{ else }}good{{ end }}'

opt/04-tools/monitoring/prometheus/alerts/alerts.yml

@@ -100,14 +100,14 @@ groups:
           description: "Container {{ $labels.name }} has restarted {{ $value }} times in the last 10 minutes"
 
       - alert: ContainerHighMemory
-        expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 90
-        for: 2m
+        expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 95
+        for: 5m
         labels:
           severity: warning
           container: "{{ $labels.name }}"
         annotations:
           summary: "Container {{ $labels.name }} high memory usage"
-          description: "Container {{ $labels.name }} memory usage is {{ $value | humanizePercentage }}"
+          description: "Container {{ $labels.name }} memory usage is {{ $value | humanizePercentage }} for more than 5 minutes"
 
       - alert: ContainerHighCPU
         expr: (rate(container_cpu_usage_seconds_total{name!=""}[5m])) * 100 > 80
@@ -124,13 +124,13 @@ groups:
     rules:
       - alert: WebsiteDown
         expr: probe_success{job="blackbox-ssl"} == 0
-        for: 1m
+        for: 10m
         labels:
           severity: critical
           instance: "{{ $labels.instance }}"
         annotations:
           summary: "Website {{ $labels.instance }} is DOWN"
-          description: "Website {{ $labels.instance }} has been unreachable for more than 1 minute"
+          description: "Website {{ $labels.instance }} has been unreachable for more than 10 minutes. Last probe at {{ $labels.job }}"
 
       - alert: WebsiteSlow
         expr: probe_duration_seconds{job="blackbox-ssl"} > 5