diff --git a/opt/04-tools/monitoring/alertmanager/config.yml b/opt/04-tools/monitoring/alertmanager/config.yml index 1326ee2..bfda0b9 100644 --- a/opt/04-tools/monitoring/alertmanager/config.yml +++ b/opt/04-tools/monitoring/alertmanager/config.yml @@ -2,10 +2,10 @@ global: slack_api_url: 'https://hooks.slack.com/services/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' route: - group_by: ['alertname'] - group_wait: 10s - group_interval: 10s - repeat_interval: 1h + group_by: ['alertname', 'severity', 'instance'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h receiver: 'slack-default' receivers: @@ -13,15 +13,18 @@ receivers: slack_configs: - channel: '#server-status' send_resolved: true - title: '{{ if eq .Status "firing" }}🚨{{ else }}✅{{ end }} {{ .GroupLabels.alertname }}' + title: '{{ if eq .Status "firing" }}🚨{{ else }}✅{{ end }} {{ .GroupLabels.alertname }} ({{ .GroupLabels.severity }})' text: |- {{ if eq .Status "firing" }}*🔥 Problem Detected*{{ else }}*✅ Problem Resolved*{{ end }} - - *Summary:* {{ .CommonAnnotations.summary }} - *Details:* {{ .CommonAnnotations.description }} + {{ range .Alerts }} + *Alert:* {{ .Labels.alertname }} + *Summary:* {{ .Annotations.summary }} + *Details:* {{ .Annotations.description }} + *Instance:* {{ .Labels.instance }}{{ if .Labels.container }} ({{ .Labels.container }}){{ end }} + {{ end }} *Severity:* {{ .CommonLabels.severity }} - + *Links:* - • - • + • + • color: '{{ if eq .Status "firing" }}{{ if eq .CommonLabels.severity "critical" }}danger{{ else }}warning{{ end }}{{ else }}good{{ end }}' diff --git a/opt/04-tools/monitoring/prometheus/alerts/alerts.yml b/opt/04-tools/monitoring/prometheus/alerts/alerts.yml index 666a263..43d3662 100644 --- a/opt/04-tools/monitoring/prometheus/alerts/alerts.yml +++ b/opt/04-tools/monitoring/prometheus/alerts/alerts.yml @@ -100,14 +100,14 @@ groups: description: "Container {{ $labels.name }} has restarted {{ $value }} times in the last 10 minutes" - alert: ContainerHighMemory - expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 90 - for: 2m + expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 95 + for: 5m labels: severity: warning container: "{{ $labels.name }}" annotations: summary: "Container {{ $labels.name }} high memory usage" - description: "Container {{ $labels.name }} memory usage is {{ $value | humanizePercentage }}" + description: "Container {{ $labels.name }} memory usage is {{ $value | humanizePercentage }} for more than 5 minutes" - alert: ContainerHighCPU expr: (rate(container_cpu_usage_seconds_total{name!=""}[5m])) * 100 > 80 @@ -124,13 +124,13 @@ groups: rules: - alert: WebsiteDown expr: probe_success{job="blackbox-ssl"} == 0 - for: 1m + for: 10m labels: severity: critical instance: "{{ $labels.instance }}" annotations: summary: "Website {{ $labels.instance }} is DOWN" - description: "Website {{ $labels.instance }} has been unreachable for more than 1 minute" + description: "Website {{ $labels.instance }} has been unreachable for more than 10 minutes. Last probe at {{ $labels.job }}" - alert: WebsiteSlow expr: probe_duration_seconds{job="blackbox-ssl"} > 5