feat: настроен полный алертинг Prometheus с 18 правилами
Изменения: - Добавлена секция rule_files в prometheus.yml - Расширены правила алертинга с 6 до 18 алертов - Снижены пороги для более раннего обнаружения проблем: * CPU: warning 70% (было 80%), critical 85% (было 90%) * Memory: warning 80% (было 85%), critical 90% * Disk: warning 80%, critical 90% (было 90%) * ServiceDown: 30s (было 1m) - Добавлены новые алерты: * ContainerDown - падение контейнеров * ContainerHighMemory/CPU - перегрузка контейнеров * WebsiteDown/Slow - проблемы с веб-сервисами * SSLCertificateExpiring - истечение SSL сертификатов * PostgreSQLDown/Slow - проблемы с БД Результат: - 3 группы алертов: infrastructure (12), webservices (4), database (3) - Alertmanager настроен на Slack #server-status - Каждый сбой будет детектироваться в течение 30s-3m 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
SamoilenkoVadym
parent
96ea83af29
commit
1a0001202b
2 changed files with 147 additions and 15 deletions
|
|
@ -1,19 +1,20 @@
|
|||
groups:
|
||||
- name: infrastructure
|
||||
rules:
|
||||
# ========== CPU Алерты ==========
|
||||
- alert: HighCPUUsage
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
||||
for: 5m
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 70
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "High CPU usage on {{ $labels.instance }}"
|
||||
description: "CPU usage is {{ $value | humanizePercentage }} for more than 5 minutes"
|
||||
description: "CPU usage is {{ $value | humanizePercentage }} for more than 3 minutes"
|
||||
|
||||
- alert: CriticalCPUUsage
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
||||
for: 2m
|
||||
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
instance: "{{ $labels.instance }}"
|
||||
|
|
@ -21,19 +22,31 @@ groups:
|
|||
summary: "CRITICAL: CPU usage on {{ $labels.instance }}"
|
||||
description: "CPU usage is {{ $value | humanizePercentage }} - immediate attention required"
|
||||
|
||||
# ========== Memory Алерты ==========
|
||||
- alert: HighMemoryUsage
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
|
||||
for: 5m
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 80
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "High Memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage is {{ $value | humanizePercentage }} for more than 5 minutes"
|
||||
description: "Memory usage is {{ $value | humanizePercentage }} for more than 3 minutes"
|
||||
|
||||
- alert: CriticalMemoryUsage
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "CRITICAL: Memory usage on {{ $labels.instance }}"
|
||||
description: "Memory usage is {{ $value | humanizePercentage }} - immediate attention required"
|
||||
|
||||
# ========== Disk Алерты ==========
|
||||
- alert: HighDiskUsage
|
||||
expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes)) * 100 > 90
|
||||
for: 10m
|
||||
expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes)) * 100 > 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
instance: "{{ $labels.instance }}"
|
||||
|
|
@ -42,23 +55,139 @@ groups:
|
|||
summary: "High Disk usage on {{ $labels.device }}"
|
||||
description: "Disk usage on {{ $labels.device }} is {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: CriticalDiskUsage
|
||||
expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes)) * 100 > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
instance: "{{ $labels.instance }}"
|
||||
device: "{{ $labels.device }}"
|
||||
annotations:
|
||||
summary: "CRITICAL: Disk usage on {{ $labels.device }}"
|
||||
description: "Disk usage on {{ $labels.device }} is {{ $value | humanizePercentage }} - immediate action needed"
|
||||
|
||||
# ========== Service Алерты ==========
|
||||
- alert: ServiceDown
|
||||
expr: up == 0
|
||||
for: 1m
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
job: "{{ $labels.job }}"
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "Service {{ $labels.job }} is DOWN"
|
||||
description: "Service {{ $labels.job }} on {{ $labels.instance }} has been down for more than 1 minute"
|
||||
description: "Service {{ $labels.job }} on {{ $labels.instance }} has been down for more than 30 seconds"
|
||||
|
||||
# ========== Container Алерты ==========
|
||||
- alert: ContainerDown
|
||||
expr: absent(container_last_seen{name!=""}) or (time() - container_last_seen{name!=""} > 60)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
container: "{{ $labels.name }}"
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} is DOWN"
|
||||
description: "Container {{ $labels.name }} has been down or not responding"
|
||||
|
||||
- alert: ContainerRestarted
|
||||
expr: changes(container_start_time_seconds[1h]) > 2
|
||||
expr: changes(container_start_time_seconds{name!=""}[10m]) > 1
|
||||
for: 0m
|
||||
labels:
|
||||
severity: info
|
||||
severity: warning
|
||||
container: "{{ $labels.name }}"
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} restarted"
|
||||
description: "Container {{ $labels.name }} has restarted {{ $value }} times in the last hour"
|
||||
description: "Container {{ $labels.name }} has restarted {{ $value }} times in the last 10 minutes"
|
||||
|
||||
- alert: ContainerHighMemory
|
||||
expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
container: "{{ $labels.name }}"
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} high memory usage"
|
||||
description: "Container {{ $labels.name }} memory usage is {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: ContainerHighCPU
|
||||
expr: (rate(container_cpu_usage_seconds_total{name!=""}[5m])) * 100 > 80
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
container: "{{ $labels.name }}"
|
||||
annotations:
|
||||
summary: "Container {{ $labels.name }} high CPU usage"
|
||||
description: "Container {{ $labels.name }} CPU usage is {{ $value | humanizePercentage }}"
|
||||
|
||||
# ========== Web Services Алерты ==========
|
||||
- name: webservices
|
||||
rules:
|
||||
- alert: WebsiteDown
|
||||
expr: probe_success{job="blackbox-ssl"} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "Website {{ $labels.instance }} is DOWN"
|
||||
description: "Website {{ $labels.instance }} has been unreachable for more than 1 minute"
|
||||
|
||||
- alert: WebsiteSlow
|
||||
expr: probe_duration_seconds{job="blackbox-ssl"} > 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "Website {{ $labels.instance }} is SLOW"
|
||||
description: "Website {{ $labels.instance }} response time is {{ $value }}s"
|
||||
|
||||
- alert: SSLCertificateExpiringSoon
|
||||
expr: (probe_ssl_earliest_cert_expiry{job="blackbox-ssl"} - time()) / 86400 < 14
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "SSL certificate expiring soon for {{ $labels.instance }}"
|
||||
description: "SSL certificate for {{ $labels.instance }} will expire in {{ $value }} days"
|
||||
|
||||
- alert: SSLCertificateExpiring
|
||||
expr: (probe_ssl_earliest_cert_expiry{job="blackbox-ssl"} - time()) / 86400 < 7
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
instance: "{{ $labels.instance }}"
|
||||
annotations:
|
||||
summary: "SSL certificate CRITICAL: {{ $labels.instance }}"
|
||||
description: "SSL certificate for {{ $labels.instance }} will expire in {{ $value }} days - immediate action required"
|
||||
|
||||
# ========== Database Алерты ==========
|
||||
- name: database
|
||||
rules:
|
||||
- alert: PostgreSQLDown
|
||||
expr: pg_up == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PostgreSQL is DOWN"
|
||||
description: "PostgreSQL database has been down for more than 30 seconds"
|
||||
|
||||
- alert: PostgreSQLTooManyConnections
|
||||
expr: sum(pg_stat_activity_count) > (pg_settings_max_connections * 0.8)
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PostgreSQL too many connections"
|
||||
description: "PostgreSQL has {{ $value }} connections (>80% of max)"
|
||||
|
||||
- alert: PostgreSQLSlowQueries
|
||||
expr: rate(pg_stat_activity_max_tx_duration[5m]) > 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PostgreSQL slow queries detected"
|
||||
description: "PostgreSQL has queries running longer than 5 minutes"
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ global:
|
|||
|
||||
# Оптимизация retention политики
|
||||
|
||||
rule_files:
|
||||
- '/etc/prometheus/alerts/*.yml'
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue