diff --git a/hosts/unstable/hosea/default.nix b/hosts/unstable/hosea/default.nix index 8ae22bb..4d61cf2 100644 --- a/hosts/unstable/hosea/default.nix +++ b/hosts/unstable/hosea/default.nix @@ -21,6 +21,11 @@ in ./hardware-configuration.nix ]; + age.secrets.grafana-secret-key = { + file = ../../../secrets/grafana-secret-key.age; + owner = "grafana"; + }; + # Bootloader boot = { loader = { @@ -124,7 +129,7 @@ in ]; }; settings = { - security.secret_key = "123456789"; + security.secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}"; server = { domain = "${config.networking.hostName}.shire-zebra.ts.net"; enforce_domain = true; diff --git a/manifests/monitoring/alerts.yaml b/manifests/monitoring/alerts.yaml new file mode 100644 index 0000000..4a90f8d --- /dev/null +++ b/manifests/monitoring/alerts.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-alerts +data: + alerts.yml: | + groups: + - name: host_health + rules: + - alert: HostDown + expr: up{job="node"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Host {{ $labels.instance }} is unreachable" + description: "Node exporter on {{ $labels.instance }} has been unreachable for more than 2 minutes." + + - alert: HighCPULoad + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage on {{ $labels.instance }} has been above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)." + + - alert: LowDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space on {{ $labels.instance }}" + description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% free ({{ $value | printf \"%.1f\" }}% remaining)." + + - alert: CriticalDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 5 + for: 2m + labels: + severity: critical + annotations: + summary: "Critical disk space on {{ $labels.instance }}" + description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 5% free ({{ $value | printf \"%.1f\" }}% remaining)." + + - alert: HighMemoryUsage + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "High memory usage on {{ $labels.instance }}" + description: "Memory usage on {{ $labels.instance }} is above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)." + + - name: backup_health + rules: + - alert: ResticBackupStaleness + # restic_last_run_success{repository=...} is exported by the prometheus graphite bridge from nas1 + # Alert if any backup job hasn't succeeded in the last 26 hours + expr: time() - restic_last_success_timestamp_seconds > 93600 + for: 0m + labels: + severity: warning + annotations: + summary: "Backup stale for {{ $labels.repository }}" + description: "Restic backup job '{{ $labels.repository }}' has not succeeded in over 26 hours." + + - name: kubernetes + rules: + - alert: PodCrashLooping + expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash-looping" + description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has restarted {{ $value | printf \"%.0f\" }} times in the last 15 minutes." + + - alert: PodNotReady + expr: kube_pod_status_ready{condition="true"} == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} not ready" + description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has not been ready for more than 10 minutes." diff --git a/manifests/monitoring/config.yaml b/manifests/monitoring/config.yaml index 144f42f..424f430 100644 --- a/manifests/monitoring/config.yaml +++ b/manifests/monitoring/config.yaml @@ -8,6 +8,9 @@ data: scrape_interval: 10s evaluation_interval: 10s alerting: + alertmanagers: [] + rule_files: + - /etc/prometheus/alerts.yml scrape_configs: # This scrapes endpoints on my servers - job_name: node @@ -20,7 +23,6 @@ data: - "hosea.shire-zebra.ts.net:9100" - "exodus.shire-zebra.ts.net:9100" - "linode.shire-zebra.ts.net:9100" - - "vm-gitlab.shire-zebra.ts.net:9100" - job_name: systemd static_configs: - targets: @@ -31,7 +33,6 @@ data: - "hosea.shire-zebra.ts.net:9558" - "exodus.shire-zebra.ts.net:9558" - "linode.shire-zebra.ts.net:9558" - - "vm-gitlab.shire-zebra.ts.net:9558" - job_name: ping static_configs: - targets: @@ -42,7 +43,6 @@ data: - "hosea.shire-zebra.ts.net:9427" - "exodus.shire-zebra.ts.net:9427" - "linode.shire-zebra.ts.net:9427" - - "vm-gitlab.shire-zebra.ts.net:9427" - job_name: kea static_configs: - targets: diff --git a/manifests/monitoring/deployment.yaml b/manifests/monitoring/deployment.yaml index b31d7b5..eaf4b4f 100644 --- a/manifests/monitoring/deployment.yaml +++ b/manifests/monitoring/deployment.yaml @@ -34,7 +34,11 @@ spec: containerPort: 9090 volumeMounts: - name: prometheus-config-values - mountPath: /etc/prometheus + mountPath: /etc/prometheus/prometheus.yml + subPath: prometheus.yml + - name: prometheus-alerts + mountPath: /etc/prometheus/alerts.yml + subPath: alerts.yml - name: prometheus-storage-volume mountPath: /prometheus restartPolicy: Always @@ -43,6 +47,10 @@ spec: configMap: defaultMode: 420 name: prometheus-config + - name: prometheus-alerts + configMap: + defaultMode: 420 + name: prometheus-alerts - name: prometheus-storage-volume persistentVolumeClaim: claimName: prometheus-data diff --git a/manifests/monitoring/kustomization.yaml b/manifests/monitoring/kustomization.yaml index cff2df8..585e437 100644 --- a/manifests/monitoring/kustomization.yaml +++ b/manifests/monitoring/kustomization.yaml @@ -5,6 +5,7 @@ resources: - pvc.yaml - cluster-role.yaml - config.yaml + - alerts.yaml - deployment.yaml - service.yaml - ingress.yaml diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 26fb152..ec83130 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -90,6 +90,8 @@ in "restic-env.age".publicKeys = everyone; "restic-pw.age".publicKeys = everyone; + "grafana-secret-key.age".publicKeys = everyone; + "dendrite.age".publicKeys = everyone; "dendrite_key.age".publicKeys = everyone;