diff --git a/opt/04-tools/monitoring/prometheus/alerts/alerts.yml b/opt/04-tools/monitoring/prometheus/alerts/alerts.yml index e97a79c..666a263 100644 --- a/opt/04-tools/monitoring/prometheus/alerts/alerts.yml +++ b/opt/04-tools/monitoring/prometheus/alerts/alerts.yml @@ -1,19 +1,20 @@ groups: - name: infrastructure rules: + # ========== CPU Алерты ========== - alert: HighCPUUsage - expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 - for: 5m + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 70 + for: 3m labels: severity: warning instance: "{{ $labels.instance }}" annotations: summary: "High CPU usage on {{ $labels.instance }}" - description: "CPU usage is {{ $value | humanizePercentage }} for more than 5 minutes" + description: "CPU usage is {{ $value | humanizePercentage }} for more than 3 minutes" - alert: CriticalCPUUsage - expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 - for: 2m + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85 + for: 1m labels: severity: critical instance: "{{ $labels.instance }}" @@ -21,19 +22,31 @@ groups: summary: "CRITICAL: CPU usage on {{ $labels.instance }}" description: "CPU usage is {{ $value | humanizePercentage }} - immediate attention required" + # ========== Memory Алерты ========== - alert: HighMemoryUsage - expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 - for: 5m + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 80 + for: 3m labels: severity: warning instance: "{{ $labels.instance }}" annotations: summary: "High Memory usage on {{ $labels.instance }}" - description: "Memory usage is {{ $value | humanizePercentage }} for more than 5 minutes" + description: "Memory usage is {{ $value | humanizePercentage }} for more than 3 minutes" + - alert: CriticalMemoryUsage + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90 + for: 1m + labels: + severity: critical + instance: "{{ $labels.instance }}" + annotations: + summary: "CRITICAL: Memory usage on {{ $labels.instance }}" + description: "Memory usage is {{ $value | humanizePercentage }} - immediate attention required" + + # ========== Disk Алерты ========== - alert: HighDiskUsage - expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes)) * 100 > 90 - for: 10m + expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes)) * 100 > 80 + for: 5m labels: severity: warning instance: "{{ $labels.instance }}" @@ -42,23 +55,139 @@ groups: summary: "High Disk usage on {{ $labels.device }}" description: "Disk usage on {{ $labels.device }} is {{ $value | humanizePercentage }}" + - alert: CriticalDiskUsage + expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes)) * 100 > 90 + for: 2m + labels: + severity: critical + instance: "{{ $labels.instance }}" + device: "{{ $labels.device }}" + annotations: + summary: "CRITICAL: Disk usage on {{ $labels.device }}" + description: "Disk usage on {{ $labels.device }} is {{ $value | humanizePercentage }} - immediate action needed" + + # ========== Service Алерты ========== - alert: ServiceDown expr: up == 0 - for: 1m + for: 30s labels: severity: critical job: "{{ $labels.job }}" instance: "{{ $labels.instance }}" annotations: summary: "Service {{ $labels.job }} is DOWN" - description: "Service {{ $labels.job }} on {{ $labels.instance }} has been down for more than 1 minute" + description: "Service {{ $labels.job }} on {{ $labels.instance }} has been down for more than 30 seconds" + + # ========== Container Алерты ========== + - alert: ContainerDown + expr: absent(container_last_seen{name!=""}) or (time() - container_last_seen{name!=""} > 60) + for: 1m + labels: + severity: critical + container: "{{ $labels.name }}" + annotations: + summary: "Container {{ $labels.name }} is DOWN" + description: "Container {{ $labels.name }} has been down or not responding" - alert: ContainerRestarted - expr: changes(container_start_time_seconds[1h]) > 2 + expr: changes(container_start_time_seconds{name!=""}[10m]) > 1 for: 0m labels: - severity: info + severity: warning container: "{{ $labels.name }}" annotations: summary: "Container {{ $labels.name }} restarted" - description: "Container {{ $labels.name }} has restarted {{ $value }} times in the last hour" + description: "Container {{ $labels.name }} has restarted {{ $value }} times in the last 10 minutes" + + - alert: ContainerHighMemory + expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 90 + for: 2m + labels: + severity: warning + container: "{{ $labels.name }}" + annotations: + summary: "Container {{ $labels.name }} high memory usage" + description: "Container {{ $labels.name }} memory usage is {{ $value | humanizePercentage }}" + + - alert: ContainerHighCPU + expr: (rate(container_cpu_usage_seconds_total{name!=""}[5m])) * 100 > 80 + for: 3m + labels: + severity: warning + container: "{{ $labels.name }}" + annotations: + summary: "Container {{ $labels.name }} high CPU usage" + description: "Container {{ $labels.name }} CPU usage is {{ $value | humanizePercentage }}" + + # ========== Web Services Алерты ========== + - name: webservices + rules: + - alert: WebsiteDown + expr: probe_success{job="blackbox-ssl"} == 0 + for: 1m + labels: + severity: critical + instance: "{{ $labels.instance }}" + annotations: + summary: "Website {{ $labels.instance }} is DOWN" + description: "Website {{ $labels.instance }} has been unreachable for more than 1 minute" + + - alert: WebsiteSlow + expr: probe_duration_seconds{job="blackbox-ssl"} > 5 + for: 2m + labels: + severity: warning + instance: "{{ $labels.instance }}" + annotations: + summary: "Website {{ $labels.instance }} is SLOW" + description: "Website {{ $labels.instance }} response time is {{ $value }}s" + + - alert: SSLCertificateExpiringSoon + expr: (probe_ssl_earliest_cert_expiry{job="blackbox-ssl"} - time()) / 86400 < 14 + for: 1h + labels: + severity: warning + instance: "{{ $labels.instance }}" + annotations: + summary: "SSL certificate expiring soon for {{ $labels.instance }}" + description: "SSL certificate for {{ $labels.instance }} will expire in {{ $value }} days" + + - alert: SSLCertificateExpiring + expr: (probe_ssl_earliest_cert_expiry{job="blackbox-ssl"} - time()) / 86400 < 7 + for: 1h + labels: + severity: critical + instance: "{{ $labels.instance }}" + annotations: + summary: "SSL certificate CRITICAL: {{ $labels.instance }}" + description: "SSL certificate for {{ $labels.instance }} will expire in {{ $value }} days - immediate action required" + + # ========== Database Алерты ========== + - name: database + rules: + - alert: PostgreSQLDown + expr: pg_up == 0 + for: 30s + labels: + severity: critical + annotations: + summary: "PostgreSQL is DOWN" + description: "PostgreSQL database has been down for more than 30 seconds" + + - alert: PostgreSQLTooManyConnections + expr: sum(pg_stat_activity_count) > (pg_settings_max_connections * 0.8) + for: 2m + labels: + severity: warning + annotations: + summary: "PostgreSQL too many connections" + description: "PostgreSQL has {{ $value }} connections (>80% of max)" + + - alert: PostgreSQLSlowQueries + expr: rate(pg_stat_activity_max_tx_duration[5m]) > 300 + for: 2m + labels: + severity: warning + annotations: + summary: "PostgreSQL slow queries detected" + description: "PostgreSQL has queries running longer than 5 minutes" diff --git a/opt/04-tools/monitoring/prometheus/prometheus.yml b/opt/04-tools/monitoring/prometheus/prometheus.yml index 84ed110..f6e6245 100644 --- a/opt/04-tools/monitoring/prometheus/prometheus.yml +++ b/opt/04-tools/monitoring/prometheus/prometheus.yml @@ -5,6 +5,9 @@ global: # Оптимизация retention политики +rule_files: + - '/etc/prometheus/alerts/*.yml' + alerting: alertmanagers: - static_configs: