alerts...
This commit is contained in:
parent
7278922625
commit
fc9971ddc9
5 changed files with 324 additions and 4 deletions
265
nixos/hosts/monitoring/provisioning/alerting/alerts.yml
Normal file
265
nixos/hosts/monitoring/provisioning/alerting/alerts.yml
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- name: node-alerts
|
||||
folder: "VM Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: high_cpu
|
||||
title: High CPU Usage
|
||||
condition: A
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "High CPU on {{ $labels.instance }}"
|
||||
description: "CPU > 90% for 5 minutes"
|
||||
labels:
|
||||
severity: warning
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: avg(rate(node_cpu_seconds_total{mode!="idle"}[5m])) by (instance) > 0.9
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: false
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- uid: high_memory
|
||||
title: High Memory Usage
|
||||
condition: A
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "High memory on {{ $labels.instance }}"
|
||||
description: "Memory > 90% for 5 minutes"
|
||||
labels:
|
||||
severity: warning
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: false
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- uid: low_disk
|
||||
title: Low Disk Space
|
||||
condition: A
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Low disk on {{ $labels.instance }} {{ $labels.mountpoint }}"
|
||||
description: "< 10% space left"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) < 0.1
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: false
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- uid: node_down
|
||||
title: Node is Down
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "{{ $labels.instance }} is DOWN"
|
||||
description: "Exporter is not responding"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: up == 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: true
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: prometheus-alerts
|
||||
folder: "Prometheus Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: prometheus_down
|
||||
title: Prometheus is Down
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Prometheus is not responding"
|
||||
description: "up{job='prometheus'} == 0"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: up{job="prometheus"} == 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: true
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: traefik-alerts
|
||||
folder: "Traefik Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: traefik_5xx
|
||||
title: Traefik 5xx Errors
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Traefik has a high rate of 5xx responses"
|
||||
description: "Rate of HTTP 5xx responses is high"
|
||||
labels:
|
||||
severity: warning
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: rate(traefik_service_requests_total{code=~"5.."}[1m]) > 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: false
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: grafana-alerts
|
||||
folder: "Grafana Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: grafana_down
|
||||
title: Grafana is Down
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Grafana is not responding"
|
||||
description: "up{job='grafana'} == 0"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: up{job="grafana"} == 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: true
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: postgres-alerts
|
||||
folder: "Postgres Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: postgres_down
|
||||
title: Postgres is Down
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Postgres is not responding"
|
||||
description: "up{job='postgres'} == 0"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: up{job="postgres"} == 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: true
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: gitea-alerts
|
||||
folder: "Gitea Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: gitea_down
|
||||
title: Gitea is Down
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Gitea is not responding"
|
||||
description: "up{job='gitea'} == 0"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: up{job="gitea"} == 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: true
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: promtail-alerts
|
||||
folder: "Promtail Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: promtail_down
|
||||
title: Promtail is Down
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Promtail is not responding"
|
||||
description: "up{job='promtail'} == 0"
|
||||
labels:
|
||||
severity: critical
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 60, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: up{job="promtail"} == 0
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: true
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
||||
- name: logs-alerts
|
||||
folder: "Logs Alerts"
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: failed_ssh_logins
|
||||
title: Failed SSH Logins
|
||||
condition: A
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: "Too many failed SSH login attempts"
|
||||
description: "Check for brute force login attempts"
|
||||
labels:
|
||||
severity: warning
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
expr: sum(rate(failed_ssh_logins[5m])) > 5
|
||||
interval: ""
|
||||
datasource: { type: prometheus, uid: prometheus }
|
||||
instant: false
|
||||
intervalMs: 15000
|
||||
maxDataPoints: 43200
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# /etc/grafana/provisioning/notifiers/contact-points.yml
|
||||
apiVersion: 1
|
||||
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: telegram
|
||||
type: telegram
|
||||
settings:
|
||||
bottoken: "__YOUR_BOT_TOKEN__"
|
||||
chatid: "__YOUR_CHAT_ID__"
|
||||
disableResolveMessage: false
|
||||
Loading…
Add table
Add a link
Reference in a new issue