Merge pull request 'fix: security and monitoring improvements (issues 1-3)' (#1) from klaatu/nixos:fix/security-and-monitoring into main

Reviewed-on: https://src.thehellings.com/greg/nixos/pulls/1

hosts/unstable/hosea/default.nix

@@ -21,6 +21,11 @@ in
     ./hardware-configuration.nix
   ];
 
+  age.secrets.grafana-secret-key = {
+    file = ../../../secrets/grafana-secret-key.age;
+    owner = "grafana";
+  };
+
   # Bootloader
   boot = {
     loader = {
@@ -124,7 +129,7 @@ in
         ];
       };
       settings = {
-        security.secret_key = "123456789";
+        security.secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}";
         server = {
           domain = "${config.networking.hostName}.shire-zebra.ts.net";
           enforce_domain = true;

manifests/monitoring/alerts.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-alerts
+data:
+  alerts.yml: |
+    groups:
+      - name: host_health
+        rules:
+          - alert: HostDown
+            expr: up{job="node"} == 0
+            for: 2m
+            labels:
+              severity: critical
+            annotations:
+              summary: "Host {{ $labels.instance }} is unreachable"
+              description: "Node exporter on {{ $labels.instance }} has been unreachable for more than 2 minutes."
+
+          - alert: HighCPULoad
+            expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "High CPU on {{ $labels.instance }}"
+              description: "CPU usage on {{ $labels.instance }} has been above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)."
+
+          - alert: LowDiskSpace
+            expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 10
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Low disk space on {{ $labels.instance }}"
+              description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% free ({{ $value | printf \"%.1f\" }}% remaining)."
+
+          - alert: CriticalDiskSpace
+            expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 5
+            for: 2m
+            labels:
+              severity: critical
+            annotations:
+              summary: "Critical disk space on {{ $labels.instance }}"
+              description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 5% free ({{ $value | printf \"%.1f\" }}% remaining)."
+
+          - alert: HighMemoryUsage
+            expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "High memory usage on {{ $labels.instance }}"
+              description: "Memory usage on {{ $labels.instance }} is above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)."
+
+      - name: backup_health
+        rules:
+          - alert: ResticBackupStaleness
+            # restic_last_run_success{repository=...} is exported by the prometheus graphite bridge from nas1
+            # Alert if any backup job hasn't succeeded in the last 26 hours
+            expr: time() - restic_last_success_timestamp_seconds > 93600
+            for: 0m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Backup stale for {{ $labels.repository }}"
+              description: "Restic backup job '{{ $labels.repository }}' has not succeeded in over 26 hours."
+
+      - name: kubernetes
+        rules:
+          - alert: PodCrashLooping
+            expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0
+            for: 5m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash-looping"
+              description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has restarted {{ $value | printf \"%.0f\" }} times in the last 15 minutes."
+
+          - alert: PodNotReady
+            expr: kube_pod_status_ready{condition="true"} == 0
+            for: 10m
+            labels:
+              severity: warning
+            annotations:
+              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} not ready"
+              description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has not been ready for more than 10 minutes."

manifests/monitoring/config.yaml

@@ -8,6 +8,9 @@ data:
       scrape_interval: 10s
       evaluation_interval: 10s
     alerting:
+      alertmanagers: []
+    rule_files:
+      - /etc/prometheus/alerts.yml
     scrape_configs:
       # This scrapes endpoints on my servers
       - job_name: node
@@ -20,7 +23,6 @@ data:
               - "hosea.shire-zebra.ts.net:9100"
               - "exodus.shire-zebra.ts.net:9100"
               - "linode.shire-zebra.ts.net:9100"
-              - "vm-gitlab.shire-zebra.ts.net:9100"
       - job_name: systemd
         static_configs:
           - targets:
@@ -31,7 +33,6 @@ data:
               - "hosea.shire-zebra.ts.net:9558"
               - "exodus.shire-zebra.ts.net:9558"
               - "linode.shire-zebra.ts.net:9558"
-              - "vm-gitlab.shire-zebra.ts.net:9558"
       - job_name: ping
         static_configs:
           - targets:
@@ -42,7 +43,6 @@ data:
               - "hosea.shire-zebra.ts.net:9427"
               - "exodus.shire-zebra.ts.net:9427"
               - "linode.shire-zebra.ts.net:9427"
-              - "vm-gitlab.shire-zebra.ts.net:9427"
       - job_name: kea
         static_configs:
           - targets:

manifests/monitoring/deployment.yaml

@@ -34,7 +34,11 @@ spec:
               containerPort: 9090
           volumeMounts:
             - name: prometheus-config-values
-              mountPath: /etc/prometheus
+              mountPath: /etc/prometheus/prometheus.yml
+              subPath: prometheus.yml
+            - name: prometheus-alerts
+              mountPath: /etc/prometheus/alerts.yml
+              subPath: alerts.yml
             - name: prometheus-storage-volume
               mountPath: /prometheus
       restartPolicy: Always
@@ -43,6 +47,10 @@ spec:
           configMap:
             defaultMode: 420
             name: prometheus-config
+        - name: prometheus-alerts
+          configMap:
+            defaultMode: 420
+            name: prometheus-alerts
         - name: prometheus-storage-volume
           persistentVolumeClaim:
             claimName: prometheus-data

manifests/monitoring/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
   - pvc.yaml
   - cluster-role.yaml
   - config.yaml
+  - alerts.yaml
   - deployment.yaml
   - service.yaml
   - ingress.yaml

secrets/grafana-secret-key.age

@@ -0,0 +1,37 @@
+age-encryption.org/v1
+-> ssh-ed25519 87huqg X/WiWcKfKUrySxBYaS50hMWmFCIkZZo85Oa5mS0H2gU
+TTPstszz10pk4Q7/BZ4vmxl7EJqlWLWuIAj78CYw6gA
+-> ssh-ed25519 8UnW5Q l6Y2VjFtX3BQzFSMoat05G8PWvoMUs20YxEfNgiqODU
+7jRxwa2ot+RjEncbSasPA0QyN5rJ/9KcBlFeJSrCUzY
+-> ssh-ed25519 UFfTmg 5VEvpZVQuVGctJ++fiBCOFNxfep9HPou/b1RDmerfA4
+5tw66zAndyRzCXiR75hQQFL2BwWfPvnbm/YfdbKUdLI
+-> ssh-ed25519 xNtnoA lDnNKjP+4LmC69L+lS7ItKXlL6iHbI+z0+S208TdJks
+HViAVNAqUpcJah4gQzfAAlpfe4CvlRDkKyKyAHN+sJQ
+-> ssh-ed25519 aY2AXA 0m5snhqLLevXntHbcLbXeZPYleCGrj5GmMGYw7o/1AY
+X6p3+DCLURhShWemK8t6hqZaByAvbqsuYNqQ0cJ7Fs0
+-> ssh-ed25519 AQhf1g kkNoHO53xVmTj8qwbz9VECJ7jVyZCgqgLDvoOZP9mmA
+/ZZA3Uhz0W3mtY4/4G/HVF4gz2MtsL8C4Mz2OcnPysE
+-> ssh-ed25519 mOmPfg EdeLqgc0EvGowYkuF4436C8o+4we2FnlaxfPqd9mbDU
+cQooIOjuGT/i/QbV4lYIP8QxWxtX49mZJpBlrtRJftw
+-> ssh-ed25519 YJiRbw I6Wk+Bq9U9yMw0fSHG+vjxXQ6vyeIoqv2Dw4LS1EFj0
+Jvt0gujTYp5kYGew3TwN8uuqRfZH+KyWyXajMhHfb5Q
+-> ssh-ed25519 0/WsKg 2mO7BzVlAlHIsq6PjnsXPlDeX5GTJ7h3LxOqnYMi4X4
+Opxp2EZgTrV3/SNlRXcU432eIz9EZMuZIkfwzUuqHb8
+-> ssh-ed25519 Nl/5yA ZTXjKkQqcAr0qgUPoPmjFeBBkwTkxdpsQUNAYZ3Fih0
+ekZbDidS5UHhvfSqPqUlgfgFz4u8bdDx4IPFWj6YI3Y
+-> ssh-ed25519 GdLgCQ HSzv8LUC0hX+pN4jKsTcGekrG1jVO+l4yf8HIT5o7Qs
+HRWZikq9o1/3BfUmYhAQtYjo72lWPeHTCcJejR+KehM
+-> ssh-ed25519 tOH/HQ X5ovKDga8yPDrstzgB2AQHCmIoyLKmR9fFWMI7OoBTA
+Hn9l2qk/nYeZVxv/3pE65jMe5gBCSHolHMgKgqZcl8M
+-> ssh-ed25519 FpzvfQ oXPL8Xv/KKZpyhEsCDBoBe3e1SUrdBweqbrO3zX8AwA
+UODBP2UzgHPSbtBMwIfZlK7QgEYT15BOD6zVhZfUby4
+-> ssh-ed25519 kdPvzQ xdXBsTF/bhyUwerBh0M0jTMXiB2jlvqbaLOGXlerMQg
+rErtHvD21aB7AuHOVTjdFq08xn5BSYxGyyd41F9nNcE
+-> ssh-ed25519 onmXpg /TJ0Lk+CyArYKZZO7UHqJExiTiCYwHSK0YphGXyzBnw
+1GiYuhSSIieuCVOGYNHZyNEoJlykokHG/7cHQbfpy88
+-> ssh-ed25519 CnhD0g /siurjKr3wuF8aPFnS9tKTbzuBIYjV7HehUQt3L1QjU
+JHXhMGkLcUo+jxtsxM8TMei7hXcappY1AKbCGT2v73U
+-> ssh-ed25519 4ep2UA UAgBtNaV/A5RAEze4X2zky5s+TgHIQ5ufHCx8neJZC8
+evGC3PCZxfIrmWj0ERxO+AVFZyQuharOLYxzLkt2NsU
+--- XEXFLANMMaS8UXRrZh3crxuY68pP3pH+BEEDP+G/hYE
+{Qá*+ŽloÞø�yâoXV
tÔt`£ÓFcO¹ìSCÓ½
ôšêÀMíù÷ˆ'’ö3ˆVý›hŒÃøô±p&‚FYínª«¼ßzö”¬%¡¿¤·šÈ%Sa¶˜;
v�dÎ8Þ

secrets/secrets.nix

@@ -90,6 +90,8 @@ in
   "restic-env.age".publicKeys = everyone;
   "restic-pw.age".publicKeys = everyone;
 
+  "grafana-secret-key.age".publicKeys = everyone;
+
   "dendrite.age".publicKeys = everyone;
   "dendrite_key.age".publicKeys = everyone;