From 7fee8824880505716a438f711da38c8da2464011 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Mar 2026 06:02:17 -0500 Subject: [PATCH 1/2] fix: security and monitoring improvements Issue 1 - Grafana secret key: - Replace hardcoded '123456789' secret_key in hosea Grafana config with an agenix-managed secret ($__file interpolation) - Add age.secrets.grafana-secret-key declaration to hosea config - Register grafana-secret-key.age in secrets/secrets.nix (publicKeys=everyone) - NOTE: The .age file still needs to be created by running: agenix -e secrets/grafana-secret-key.age and entering a strong random secret (e.g. from 'pwgen -s 64 1') Issue 2 - Prometheus stale targets: - Remove retired vm-gitlab host from all three scrape jobs (node, systemd, ping) in manifests/monitoring/config.yaml - linode was already present in all scrape jobs Issue 3 - Prometheus alerting gaps: - Wire up rule_files in prometheus.yml pointing at alerts.yml - Add manifests/monitoring/alerts.yaml with alerting rules: * HostDown (node unreachable > 2m) - critical * HighCPULoad (>90% for 10m) - warning * LowDiskSpace (<10% free) - warning * CriticalDiskSpace (<5% free) - critical * HighMemoryUsage (>90% for 10m) - warning * ResticBackupStaleness (no success > 26h) - warning * PodCrashLooping - warning * PodNotReady (>10m) - warning - Mount alerts ConfigMap as separate volume subPath in deployment so prometheus.yml and alerts.yml are both accessible at /etc/prometheus/ without conflicts --- hosts/unstable/hosea/default.nix | 7 +- manifests/monitoring/alerts.yaml | 86 +++++++++++++++++++++++++ manifests/monitoring/config.yaml | 6 +- manifests/monitoring/deployment.yaml | 10 ++- manifests/monitoring/kustomization.yaml | 1 + secrets/secrets.nix | 2 + 6 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 manifests/monitoring/alerts.yaml diff --git a/hosts/unstable/hosea/default.nix b/hosts/unstable/hosea/default.nix index 8ae22bb..4d61cf2 100644 --- a/hosts/unstable/hosea/default.nix +++ b/hosts/unstable/hosea/default.nix @@ -21,6 +21,11 @@ in ./hardware-configuration.nix ]; + age.secrets.grafana-secret-key = { + file = ../../../secrets/grafana-secret-key.age; + owner = "grafana"; + }; + # Bootloader boot = { loader = { @@ -124,7 +129,7 @@ in ]; }; settings = { - security.secret_key = "123456789"; + security.secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}"; server = { domain = "${config.networking.hostName}.shire-zebra.ts.net"; enforce_domain = true; diff --git a/manifests/monitoring/alerts.yaml b/manifests/monitoring/alerts.yaml new file mode 100644 index 0000000..4a90f8d --- /dev/null +++ b/manifests/monitoring/alerts.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-alerts +data: + alerts.yml: | + groups: + - name: host_health + rules: + - alert: HostDown + expr: up{job="node"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Host {{ $labels.instance }} is unreachable" + description: "Node exporter on {{ $labels.instance }} has been unreachable for more than 2 minutes." + + - alert: HighCPULoad + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU on {{ $labels.instance }}" + description: "CPU usage on {{ $labels.instance }} has been above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)." + + - alert: LowDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space on {{ $labels.instance }}" + description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 10% free ({{ $value | printf \"%.1f\" }}% remaining)." + + - alert: CriticalDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"}) * 100 < 5 + for: 2m + labels: + severity: critical + annotations: + summary: "Critical disk space on {{ $labels.instance }}" + description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has less than 5% free ({{ $value | printf \"%.1f\" }}% remaining)." + + - alert: HighMemoryUsage + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "High memory usage on {{ $labels.instance }}" + description: "Memory usage on {{ $labels.instance }} is above 90% for 10 minutes (current: {{ $value | printf \"%.1f\" }}%)." + + - name: backup_health + rules: + - alert: ResticBackupStaleness + # restic_last_run_success{repository=...} is exported by the prometheus graphite bridge from nas1 + # Alert if any backup job hasn't succeeded in the last 26 hours + expr: time() - restic_last_success_timestamp_seconds > 93600 + for: 0m + labels: + severity: warning + annotations: + summary: "Backup stale for {{ $labels.repository }}" + description: "Restic backup job '{{ $labels.repository }}' has not succeeded in over 26 hours." + + - name: kubernetes + rules: + - alert: PodCrashLooping + expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 15 > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash-looping" + description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has restarted {{ $value | printf \"%.0f\" }} times in the last 15 minutes." + + - alert: PodNotReady + expr: kube_pod_status_ready{condition="true"} == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} not ready" + description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has not been ready for more than 10 minutes." diff --git a/manifests/monitoring/config.yaml b/manifests/monitoring/config.yaml index 144f42f..424f430 100644 --- a/manifests/monitoring/config.yaml +++ b/manifests/monitoring/config.yaml @@ -8,6 +8,9 @@ data: scrape_interval: 10s evaluation_interval: 10s alerting: + alertmanagers: [] + rule_files: + - /etc/prometheus/alerts.yml scrape_configs: # This scrapes endpoints on my servers - job_name: node @@ -20,7 +23,6 @@ data: - "hosea.shire-zebra.ts.net:9100" - "exodus.shire-zebra.ts.net:9100" - "linode.shire-zebra.ts.net:9100" - - "vm-gitlab.shire-zebra.ts.net:9100" - job_name: systemd static_configs: - targets: @@ -31,7 +33,6 @@ data: - "hosea.shire-zebra.ts.net:9558" - "exodus.shire-zebra.ts.net:9558" - "linode.shire-zebra.ts.net:9558" - - "vm-gitlab.shire-zebra.ts.net:9558" - job_name: ping static_configs: - targets: @@ -42,7 +43,6 @@ data: - "hosea.shire-zebra.ts.net:9427" - "exodus.shire-zebra.ts.net:9427" - "linode.shire-zebra.ts.net:9427" - - "vm-gitlab.shire-zebra.ts.net:9427" - job_name: kea static_configs: - targets: diff --git a/manifests/monitoring/deployment.yaml b/manifests/monitoring/deployment.yaml index b31d7b5..eaf4b4f 100644 --- a/manifests/monitoring/deployment.yaml +++ b/manifests/monitoring/deployment.yaml @@ -34,7 +34,11 @@ spec: containerPort: 9090 volumeMounts: - name: prometheus-config-values - mountPath: /etc/prometheus + mountPath: /etc/prometheus/prometheus.yml + subPath: prometheus.yml + - name: prometheus-alerts + mountPath: /etc/prometheus/alerts.yml + subPath: alerts.yml - name: prometheus-storage-volume mountPath: /prometheus restartPolicy: Always @@ -43,6 +47,10 @@ spec: configMap: defaultMode: 420 name: prometheus-config + - name: prometheus-alerts + configMap: + defaultMode: 420 + name: prometheus-alerts - name: prometheus-storage-volume persistentVolumeClaim: claimName: prometheus-data diff --git a/manifests/monitoring/kustomization.yaml b/manifests/monitoring/kustomization.yaml index cff2df8..585e437 100644 --- a/manifests/monitoring/kustomization.yaml +++ b/manifests/monitoring/kustomization.yaml @@ -5,6 +5,7 @@ resources: - pvc.yaml - cluster-role.yaml - config.yaml + - alerts.yaml - deployment.yaml - service.yaml - ingress.yaml diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 26fb152..ec83130 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -90,6 +90,8 @@ in "restic-env.age".publicKeys = everyone; "restic-pw.age".publicKeys = everyone; + "grafana-secret-key.age".publicKeys = everyone; + "dendrite.age".publicKeys = everyone; "dendrite_key.age".publicKeys = everyone; From 69ebf78d6cb508af166938743131e88396f1c5e2 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 25 Mar 2026 06:35:20 -0500 Subject: [PATCH 2/2] fix: add new agenix file --- secrets/grafana-secret-key.age | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 secrets/grafana-secret-key.age diff --git a/secrets/grafana-secret-key.age b/secrets/grafana-secret-key.age new file mode 100644 index 0000000..ede9ff3 --- /dev/null +++ b/secrets/grafana-secret-key.age @@ -0,0 +1,37 @@ +age-encryption.org/v1 +-> ssh-ed25519 87huqg X/WiWcKfKUrySxBYaS50hMWmFCIkZZo85Oa5mS0H2gU +TTPstszz10pk4Q7/BZ4vmxl7EJqlWLWuIAj78CYw6gA +-> ssh-ed25519 8UnW5Q l6Y2VjFtX3BQzFSMoat05G8PWvoMUs20YxEfNgiqODU +7jRxwa2ot+RjEncbSasPA0QyN5rJ/9KcBlFeJSrCUzY +-> ssh-ed25519 UFfTmg 5VEvpZVQuVGctJ++fiBCOFNxfep9HPou/b1RDmerfA4 +5tw66zAndyRzCXiR75hQQFL2BwWfPvnbm/YfdbKUdLI +-> ssh-ed25519 xNtnoA lDnNKjP+4LmC69L+lS7ItKXlL6iHbI+z0+S208TdJks +HViAVNAqUpcJah4gQzfAAlpfe4CvlRDkKyKyAHN+sJQ +-> ssh-ed25519 aY2AXA 0m5snhqLLevXntHbcLbXeZPYleCGrj5GmMGYw7o/1AY +X6p3+DCLURhShWemK8t6hqZaByAvbqsuYNqQ0cJ7Fs0 +-> ssh-ed25519 AQhf1g kkNoHO53xVmTj8qwbz9VECJ7jVyZCgqgLDvoOZP9mmA +/ZZA3Uhz0W3mtY4/4G/HVF4gz2MtsL8C4Mz2OcnPysE +-> ssh-ed25519 mOmPfg EdeLqgc0EvGowYkuF4436C8o+4we2FnlaxfPqd9mbDU +cQooIOjuGT/i/QbV4lYIP8QxWxtX49mZJpBlrtRJftw +-> ssh-ed25519 YJiRbw I6Wk+Bq9U9yMw0fSHG+vjxXQ6vyeIoqv2Dw4LS1EFj0 +Jvt0gujTYp5kYGew3TwN8uuqRfZH+KyWyXajMhHfb5Q +-> ssh-ed25519 0/WsKg 2mO7BzVlAlHIsq6PjnsXPlDeX5GTJ7h3LxOqnYMi4X4 +Opxp2EZgTrV3/SNlRXcU432eIz9EZMuZIkfwzUuqHb8 +-> ssh-ed25519 Nl/5yA ZTXjKkQqcAr0qgUPoPmjFeBBkwTkxdpsQUNAYZ3Fih0 +ekZbDidS5UHhvfSqPqUlgfgFz4u8bdDx4IPFWj6YI3Y +-> ssh-ed25519 GdLgCQ HSzv8LUC0hX+pN4jKsTcGekrG1jVO+l4yf8HIT5o7Qs +HRWZikq9o1/3BfUmYhAQtYjo72lWPeHTCcJejR+KehM +-> ssh-ed25519 tOH/HQ X5ovKDga8yPDrstzgB2AQHCmIoyLKmR9fFWMI7OoBTA +Hn9l2qk/nYeZVxv/3pE65jMe5gBCSHolHMgKgqZcl8M +-> ssh-ed25519 FpzvfQ oXPL8Xv/KKZpyhEsCDBoBe3e1SUrdBweqbrO3zX8AwA +UODBP2UzgHPSbtBMwIfZlK7QgEYT15BOD6zVhZfUby4 +-> ssh-ed25519 kdPvzQ xdXBsTF/bhyUwerBh0M0jTMXiB2jlvqbaLOGXlerMQg +rErtHvD21aB7AuHOVTjdFq08xn5BSYxGyyd41F9nNcE +-> ssh-ed25519 onmXpg /TJ0Lk+CyArYKZZO7UHqJExiTiCYwHSK0YphGXyzBnw +1GiYuhSSIieuCVOGYNHZyNEoJlykokHG/7cHQbfpy88 +-> ssh-ed25519 CnhD0g /siurjKr3wuF8aPFnS9tKTbzuBIYjV7HehUQt3L1QjU +JHXhMGkLcUo+jxtsxM8TMei7hXcappY1AKbCGT2v73U +-> ssh-ed25519 4ep2UA UAgBtNaV/A5RAEze4X2zky5s+TgHIQ5ufHCx8neJZC8 +evGC3PCZxfIrmWj0ERxO+AVFZyQuharOLYxzLkt2NsU +--- XEXFLANMMaS8UXRrZh3crxuY68pP3pH+BEEDP+G/hYE +{Q*+loyoXVt t`FcOSCӽM'3Vhp&FYnz%%Sa;vd8 \ No newline at end of file