feat: comprehensive observability stack

- Add restic_backups and mattermost scrape jobs to Prometheus config
- Add UnPoller deployment+service+secret for UniFi metrics (unpoller.yaml)
- Add unifi scrape job targeting unpoller in-cluster
- Add 4 Grafana dashboards as NixOS-managed etc files on hosea:
  - system-health: CPU, memory, disk, network per host
  - backup-health: restic backup freshness and duration
  - kubernetes: pod counts, restart rates, node resources
  - network: DNS queries, DHCP leases, UniFi ports, ping latency
- Add grafana-api-token agenix secret placeholder on hosea
- Open firewall ports 9100/9427/9558 for prometheus exporters in baseline.nix

manifests/monitoring/config.yaml

@@ -56,6 +56,22 @@ data:
         static_configs:
           - targets:
               - "hosea.shire-zebra.ts.net:9108"
+      # Restic rest-server backup metrics
+      - job_name: restic_backups
+        static_configs:
+          - targets:
+              - "nas1.shire-zebra.ts.net:30248"
+      # Mattermost metrics
+      - job_name: mattermost
+        metrics_path: /metrics
+        static_configs:
+          - targets:
+              - "mattermost.thehellings.lan:8065"
+      # UniFi metrics via unpoller
+      - job_name: unifi
+        static_configs:
+          - targets:
+              - "unpoller.monitoring.svc.cluster.local:9130"
       # This scrapes metrics from the Kubernetes kubelets
       - job_name: kubelet
         kubernetes_sd_configs:

manifests/monitoring/kustomization.yaml

@@ -9,3 +9,4 @@ resources:
   - deployment.yaml
   - service.yaml
   - ingress.yaml
+  - unpoller.yaml

manifests/monitoring/unpoller.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: unpoller-credentials
+  namespace: monitoring
+type: Opaque
+stringData:
+  # Replace with actual UniFi controller password for the 'unpoller' read-only user
+  password: "CHANGEME"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: unpoller
+  namespace: monitoring
+  labels:
+    app: unpoller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: unpoller
+  template:
+    metadata:
+      labels:
+        app: unpoller
+    spec:
+      containers:
+        - name: unpoller
+          image: ghcr.io/unpoller/unpoller:latest
+          ports:
+            - name: metrics
+              containerPort: 9130
+          env:
+            - name: UP_UNIFI_DEFAULT_URL
+              value: "https://10.42.1.1:8443"
+            - name: UP_UNIFI_DEFAULT_VERIFY_SSL
+              value: "false"
+            - name: UP_UNIFI_DEFAULT_USER
+              value: "unpoller"
+            - name: UP_UNIFI_DEFAULT_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: unpoller-credentials
+                  key: password
+            - name: UP_PROMETHEUS_DISABLE
+              value: "false"
+            - name: UP_INFLUXDB_DISABLE
+              value: "true"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: unpoller
+  namespace: monitoring
+  labels:
+    app: unpoller
+spec:
+  selector:
+    app: unpoller
+  ports:
+    - name: metrics
+      port: 9130
+      targetPort: 9130