diff --git a/hosts/baseline.nix b/hosts/baseline.nix index 1a59328..4c70936 100644 --- a/hosts/baseline.nix +++ b/hosts/baseline.nix @@ -114,6 +114,12 @@ }; }; + networking.firewall.allowedTCPPorts = [ + 9100 # prometheus node exporter + 9427 # prometheus ping exporter + 9558 # prometheus systemd exporter + ]; + security.sudo.extraRules = [ { users = [ "greg" ]; diff --git a/hosts/unstable/hosea/default.nix b/hosts/unstable/hosea/default.nix index 4d61cf2..c110c14 100644 --- a/hosts/unstable/hosea/default.nix +++ b/hosts/unstable/hosea/default.nix @@ -148,6 +148,269 @@ in }; }; + # After first deploy: create a Grafana service account + API token for Klaatu + # via the Grafana UI, then encrypt it: agenix -e secrets/grafana-api-token.age + age.secrets.grafana-api-token = { + file = ../../../secrets/grafana-api-token.age; + owner = "grafana"; + }; + + environment.etc = { + "grafana-dashboards/system-health.json".text = '' + { + "uid": "system-health", + "title": "System Health Overview", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "time": {"from": "now-3h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Hosts Up", + "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(up{job=\"node\"} == 1)", "refId": "A"}] + }, + { + "id": 2, + "type": "stat", + "title": "Hosts Down", + "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(up{job=\"node\"} == 0) or vector(0)", "refId": "A"}], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}] + } + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "CPU Usage %", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "id": 4, + "type": "timeseries", + "title": "Memory Usage %", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "id": 5, + "type": "timeseries", + "title": "Disk Usage % (/)", + "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}) * 100)", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "id": 6, + "type": "timeseries", + "title": "Network RX bytes/s", + "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_network_receive_bytes_total{device!~\"lo|tailscale.*\"}[5m])", "legendFormat": "{{instance}} {{device}}", "refId": "A"}] + }, + { + "id": 7, + "type": "timeseries", + "title": "Network TX bytes/s", + "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_network_transmit_bytes_total{device!~\"lo|tailscale.*\"}[5m])", "legendFormat": "{{instance}} {{device}}", "refId": "A"}] + } + ] + } + ''; + + "grafana-dashboards/backup-health.json".text = '' + { + "uid": "backup-health", + "title": "Backup Health", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "time": {"from": "now-3h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Backup Jobs Reporting", + "gridPos": {"x": 0, "y": 0, "w": 8, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(restic_last_success_timestamp_seconds) or vector(0)", "refId": "A"}] + }, + { + "id": 2, + "type": "stat", + "title": "Stale Backups (>26h)", + "gridPos": {"x": 8, "y": 0, "w": 8, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(time() - restic_last_success_timestamp_seconds > 93600) or vector(0)", "refId": "A"}], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}] + } + } + } + }, + { + "id": 3, + "type": "table", + "title": "Last Backup Times", + "gridPos": {"x": 0, "y": 4, "w": 24, "h": 10}, + "options": {"instant": true}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "restic_last_success_timestamp_seconds", "instant": true, "refId": "A"}], + "fieldConfig": {"defaults": {"unit": "dateTimeAsLocal"}} + }, + { + "id": 4, + "type": "timeseries", + "title": "Backup Duration (seconds)", + "gridPos": {"x": 0, "y": 14, "w": 24, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "restic_last_run_duration_seconds", "refId": "A"}] + } + ] + } + ''; + + "grafana-dashboards/kubernetes.json".text = '' + { + "uid": "kubernetes-overview", + "title": "Kubernetes Overview", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "time": {"from": "now-3h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Running Pods", + "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Running\"}) or vector(0)", "refId": "A"}] + }, + { + "id": 2, + "type": "stat", + "title": "Failed Pods", + "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Failed\"}) or vector(0)", "refId": "A"}], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}] + } + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Pending Pods", + "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Pending\"}) or vector(0)", "refId": "A"}], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 1}] + } + } + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Pod Restart Rate", + "gridPos": {"x": 0, "y": 4, "w": 24, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "topk(10, rate(kube_pod_container_status_restarts_total[15m]) * 900)", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}] + }, + { + "id": 5, + "type": "table", + "title": "All Pods", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 10}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kube_pod_status_phase", "instant": true, "refId": "A"}] + }, + { + "id": 6, + "type": "timeseries", + "title": "Node CPU Usage", + "gridPos": {"x": 0, "y": 22, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(node) (rate(node_cpu_seconds_total{mode=\"idle\",job=\"node\"}[5m])) * 100)", "refId": "A"}] + }, + { + "id": 7, + "type": "timeseries", + "title": "Node Memory Usage %", + "gridPos": {"x": 12, "y": 22, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "refId": "A"}] + } + ] + } + ''; + + "grafana-dashboards/network.json".text = '' + { + "uid": "network-overview", + "title": "Network & UniFi", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "time": {"from": "now-3h", "to": "now"}, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "DNS Queries/s", + "gridPos": {"x": 0, "y": 0, "w": 8, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(dnsmasq_queries_total[5m]) or vector(0)", "refId": "A"}] + }, + { + "id": 2, + "type": "stat", + "title": "DHCP Leases", + "gridPos": {"x": 8, "y": 0, "w": 8, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kea_dhcp4_addresses_assigned_total or vector(0)", "refId": "A"}] + }, + { + "id": 3, + "type": "stat", + "title": "UniFi Devices", + "gridPos": {"x": 16, "y": 0, "w": 8, "h": 4}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(unifipoller_device_uptime_seconds) or vector(0)", "refId": "A"}] + }, + { + "id": 4, + "type": "timeseries", + "title": "UniFi Port RX (bytes/s)", + "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_port_receive_bytes_total[5m])", "legendFormat": "{{port_id}} {{name}}", "refId": "A"}] + }, + { + "id": 5, + "type": "timeseries", + "title": "UniFi Port TX (bytes/s)", + "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_port_transmit_bytes_total[5m])", "legendFormat": "{{port_id}} {{name}}", "refId": "A"}] + }, + { + "id": 6, + "type": "timeseries", + "title": "Ping Latency (ms)", + "gridPos": {"x": 0, "y": 12, "w": 24, "h": 8}, + "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "probe_duration_seconds{job=\"ping\"} * 1000", "legendFormat": "{{instance}}", "refId": "A"}] + } + ] + } + ''; + }; + users.users = { greg.extraGroups = [ "vboxusers" ]; diff --git a/manifests/monitoring/config.yaml b/manifests/monitoring/config.yaml index 424f430..e69af91 100644 --- a/manifests/monitoring/config.yaml +++ b/manifests/monitoring/config.yaml @@ -56,6 +56,22 @@ data: static_configs: - targets: - "hosea.shire-zebra.ts.net:9108" + # Restic rest-server backup metrics + - job_name: restic_backups + static_configs: + - targets: + - "nas1.shire-zebra.ts.net:30248" + # Mattermost metrics + - job_name: mattermost + metrics_path: /metrics + static_configs: + - targets: + - "mattermost.thehellings.lan:8065" + # UniFi metrics via unpoller + - job_name: unifi + static_configs: + - targets: + - "unpoller.monitoring.svc.cluster.local:9130" # This scrapes metrics from the Kubernetes kubelets - job_name: kubelet kubernetes_sd_configs: diff --git a/manifests/monitoring/kustomization.yaml b/manifests/monitoring/kustomization.yaml index 585e437..ebf4798 100644 --- a/manifests/monitoring/kustomization.yaml +++ b/manifests/monitoring/kustomization.yaml @@ -9,3 +9,4 @@ resources: - deployment.yaml - service.yaml - ingress.yaml + - unpoller.yaml diff --git a/manifests/monitoring/unpoller.yaml b/manifests/monitoring/unpoller.yaml new file mode 100644 index 0000000..5c83f58 --- /dev/null +++ b/manifests/monitoring/unpoller.yaml @@ -0,0 +1,65 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: unpoller-credentials + namespace: monitoring +type: Opaque +stringData: + # Replace with actual UniFi controller password for the 'unpoller' read-only user + password: "CHANGEME" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: unpoller + namespace: monitoring + labels: + app: unpoller +spec: + replicas: 1 + selector: + matchLabels: + app: unpoller + template: + metadata: + labels: + app: unpoller + spec: + containers: + - name: unpoller + image: ghcr.io/unpoller/unpoller:latest + ports: + - name: metrics + containerPort: 9130 + env: + - name: UP_UNIFI_DEFAULT_URL + value: "https://10.42.1.1:8443" + - name: UP_UNIFI_DEFAULT_VERIFY_SSL + value: "false" + - name: UP_UNIFI_DEFAULT_USER + value: "unpoller" + - name: UP_UNIFI_DEFAULT_PASS + valueFrom: + secretKeyRef: + name: unpoller-credentials + key: password + - name: UP_PROMETHEUS_DISABLE + value: "false" + - name: UP_INFLUXDB_DISABLE + value: "true" +--- +apiVersion: v1 +kind: Service +metadata: + name: unpoller + namespace: monitoring + labels: + app: unpoller +spec: + selector: + app: unpoller + ports: + - name: metrics + port: 9130 + targetPort: 9130 diff --git a/secrets/secrets.nix b/secrets/secrets.nix index ec83130..cffce51 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -135,4 +135,6 @@ in ]; "compose/attic.env.age".publicKeys = everyone; + + "grafana-api-token.age".publicKeys = everyone; }