fix: security and monitoring improvements

Issue 1 - Grafana secret key:
- Replace hardcoded '123456789' secret_key in hosea Grafana config
  with an agenix-managed secret ($__file interpolation)
- Add age.secrets.grafana-secret-key declaration to hosea config
- Register grafana-secret-key.age in secrets/secrets.nix (publicKeys=everyone)
- NOTE: The .age file still needs to be created by running:
    agenix -e secrets/grafana-secret-key.age
  and entering a strong random secret (e.g. from 'pwgen -s 64 1')

Issue 2 - Prometheus stale targets:
- Remove retired vm-gitlab host from all three scrape jobs
  (node, systemd, ping) in manifests/monitoring/config.yaml
- linode was already present in all scrape jobs

Issue 3 - Prometheus alerting gaps:
- Wire up rule_files in prometheus.yml pointing at alerts.yml
- Add manifests/monitoring/alerts.yaml with alerting rules:
    * HostDown (node unreachable > 2m) - critical
    * HighCPULoad (>90% for 10m) - warning
    * LowDiskSpace (<10% free) - warning
    * CriticalDiskSpace (<5% free) - critical
    * HighMemoryUsage (>90% for 10m) - warning
    * ResticBackupStaleness (no success > 26h) - warning
    * PodCrashLooping - warning
    * PodNotReady (>10m) - warning
- Mount alerts ConfigMap as separate volume subPath in deployment
  so prometheus.yml and alerts.yml are both accessible at
  /etc/prometheus/ without conflicts

hosts/unstable/hosea/default.nix

@@ -21,6 +21,11 @@ in
     ./hardware-configuration.nix
   ];
 
+  age.secrets.grafana-secret-key = {
+    file = ../../../secrets/grafana-secret-key.age;
+    owner = "grafana";
+  };
+
   # Bootloader
   boot = {
     loader = {
@@ -124,7 +129,7 @@ in
         ];
       };
       settings = {
-        security.secret_key = "123456789";
+        security.secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}";
         server = {
           domain = "${config.networking.hostName}.shire-zebra.ts.net";
           enforce_domain = true;

manifests/monitoring/alerts.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-alerts
+data:
+  alerts.yml: |
+    groups:
+      - name: host_health
+        rules:
+          - alert: HostDown
+            expr: up{job="node"} == 0
+            for: 2m
+            labels:
+              severity: critical
+            annotations:
+              summary: "Host {{ $labels.instance }} is unreachable"
+              description: "Node exporter on {{ $labels.instance }} has been unreachable for more than 2 minutes."
+
+          - alert: HighCPULoad
+            expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "High CPU on {{ $labels.instance }}"
+              description: "CPU usage on {{ $labels.instance }} has been above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)."
+
+          - alert: LowDiskSpace
+            expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 10
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Low disk space on {{ $labels.instance }}"
+              description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% free ({{ $value | printf \"%.1f\" }}% remaining)."
+
+          - alert: CriticalDiskSpace
+            expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 5
+            for: 2m
+            labels:
+              severity: critical
+            annotations:
+              summary: "Critical disk space on {{ $labels.instance }}"
+              description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 5% free ({{ $value | printf \"%.1f\" }}% remaining)."
+
+          - alert: HighMemoryUsage
+            expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "High memory usage on {{ $labels.instance }}"
+              description: "Memory usage on {{ $labels.instance }} is above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)."
+
+      - name: backup_health
+        rules:
+          - alert: ResticBackupStaleness
+            # restic_last_run_success{repository=...} is exported by the prometheus graphite bridge from nas1
+            # Alert if any backup job hasn't succeeded in the last 26 hours
+            expr: time() - restic_last_success_timestamp_seconds > 93600
+            for: 0m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Backup stale for {{ $labels.repository }}"
+              description: "Restic backup job '{{ $labels.repository }}' has not succeeded in over 26 hours."
+
+      - name: kubernetes
+        rules:
+          - alert: PodCrashLooping
+            expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash-looping"
+              description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has restarted {{ $value | printf \"%.0f\" }} times in the last 15 minutes."
+
+          - alert: PodNotReady
+            expr: kube_pod_status_ready{condition="true"} == 0
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} not ready"
+              description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has not been ready for more than 10 minutes."

manifests/monitoring/config.yaml

@@ -8,6 +8,9 @@ data:
       scrape_interval: 10s
       evaluation_interval: 10s
     alerting:
+      alertmanagers: []
+    rule_files:
+      - /etc/prometheus/alerts.yml
     scrape_configs:
       # This scrapes endpoints on my servers
       - job_name: node
@@ -20,7 +23,6 @@ data:
               - "hosea.shire-zebra.ts.net:9100"
               - "exodus.shire-zebra.ts.net:9100"
               - "linode.shire-zebra.ts.net:9100"
-              - "vm-gitlab.shire-zebra.ts.net:9100"
       - job_name: systemd
         static_configs:
           - targets:
@@ -31,7 +33,6 @@ data:
               - "hosea.shire-zebra.ts.net:9558"
               - "exodus.shire-zebra.ts.net:9558"
               - "linode.shire-zebra.ts.net:9558"
-              - "vm-gitlab.shire-zebra.ts.net:9558"
       - job_name: ping
         static_configs:
           - targets:
@@ -42,7 +43,6 @@ data:
               - "hosea.shire-zebra.ts.net:9427"
               - "exodus.shire-zebra.ts.net:9427"
               - "linode.shire-zebra.ts.net:9427"
-              - "vm-gitlab.shire-zebra.ts.net:9427"
       - job_name: kea
         static_configs:
           - targets:

manifests/monitoring/deployment.yaml

@@ -34,7 +34,11 @@ spec:
               containerPort: 9090
           volumeMounts:
             - name: prometheus-config-values
-              mountPath: /etc/prometheus
+              mountPath: /etc/prometheus/prometheus.yml
+              subPath: prometheus.yml
+            - name: prometheus-alerts
+              mountPath: /etc/prometheus/alerts.yml
+              subPath: alerts.yml
             - name: prometheus-storage-volume
               mountPath: /prometheus
       restartPolicy: Always
@@ -43,6 +47,10 @@ spec:
           configMap:
             defaultMode: 420
             name: prometheus-config
+        - name: prometheus-alerts
+          configMap:
+            defaultMode: 420
+            name: prometheus-alerts
         - name: prometheus-storage-volume
           persistentVolumeClaim:
             claimName: prometheus-data

manifests/monitoring/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - pvc.yaml
   - cluster-role.yaml
   - config.yaml
+  - alerts.yaml
   - deployment.yaml
   - service.yaml
   - ingress.yaml

secrets/secrets.nix

@@ -90,6 +90,8 @@ in
   "restic-env.age".publicKeys = everyone;
   "restic-pw.age".publicKeys = everyone;
 
+  "grafana-secret-key.age".publicKeys = everyone;
+
   "dendrite.age".publicKeys = everyone;
   "dendrite_key.age".publicKeys = everyone;