root
7fee882488
buildbot/nix-eval Build done.
Issue 1 - Grafana secret key:
- Replace hardcoded '123456789' secret_key in hosea Grafana config
with an agenix-managed secret ($__file interpolation)
- Add age.secrets.grafana-secret-key declaration to hosea config
- Register grafana-secret-key.age in secrets/secrets.nix (publicKeys=everyone)
- NOTE: The .age file still needs to be created by running:
agenix -e secrets/grafana-secret-key.age
and entering a strong random secret (e.g. from 'pwgen -s 64 1')
Issue 2 - Prometheus stale targets:
- Remove retired vm-gitlab host from all three scrape jobs
(node, systemd, ping) in manifests/monitoring/config.yaml
- linode was already present in all scrape jobs
Issue 3 - Prometheus alerting gaps:
- Wire up rule_files in prometheus.yml pointing at alerts.yml
- Add manifests/monitoring/alerts.yaml with alerting rules:
* HostDown (node unreachable > 2m) - critical
* HighCPULoad (>90% for 10m) - warning
* LowDiskSpace (<10% free) - warning
* CriticalDiskSpace (<5% free) - critical
* HighMemoryUsage (>90% for 10m) - warning
* ResticBackupStaleness (no success > 26h) - warning
* PodCrashLooping - warning
* PodNotReady (>10m) - warning
- Mount alerts ConfigMap as separate volume subPath in deployment
so prometheus.yml and alerts.yml are both accessible at
/etc/prometheus/ without conflicts
87 lines
3.9 KiB
YAML
87 lines
3.9 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-alerts
|
|
data:
|
|
alerts.yml: |
|
|
groups:
|
|
- name: host_health
|
|
rules:
|
|
- alert: HostDown
|
|
expr: up{job="node"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Host {{ $labels.instance }} is unreachable"
|
|
description: "Node exporter on {{ $labels.instance }} has been unreachable for more than 2 minutes."
|
|
|
|
- alert: HighCPULoad
|
|
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU on {{ $labels.instance }}"
|
|
description: "CPU usage on {{ $labels.instance }} has been above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)."
|
|
|
|
- alert: LowDiskSpace
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Low disk space on {{ $labels.instance }}"
|
|
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% free ({{ $value | printf \"%.1f\" }}% remaining)."
|
|
|
|
- alert: CriticalDiskSpace
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 5
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Critical disk space on {{ $labels.instance }}"
|
|
description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 5% free ({{ $value | printf \"%.1f\" }}% remaining)."
|
|
|
|
- alert: HighMemoryUsage
|
|
expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High memory usage on {{ $labels.instance }}"
|
|
description: "Memory usage on {{ $labels.instance }} is above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)."
|
|
|
|
- name: backup_health
|
|
rules:
|
|
- alert: ResticBackupStaleness
|
|
# restic_last_run_success{repository=...} is exported by the prometheus graphite bridge from nas1
|
|
# Alert if any backup job hasn't succeeded in the last 26 hours
|
|
expr: time() - restic_last_success_timestamp_seconds > 93600
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Backup stale for {{ $labels.repository }}"
|
|
description: "Restic backup job '{{ $labels.repository }}' has not succeeded in over 26 hours."
|
|
|
|
- name: kubernetes
|
|
rules:
|
|
- alert: PodCrashLooping
|
|
expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash-looping"
|
|
description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has restarted {{ $value | printf \"%.0f\" }} times in the last 15 minutes."
|
|
|
|
- alert: PodNotReady
|
|
expr: kube_pod_status_ready{condition="true"} == 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} not ready"
|
|
description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has not been ready for more than 10 minutes."
|