feat: update kubernetes and network Grafana dashboards

Reviewed-on: https://src.thehellings.com/greg/nixos/pulls/8

hosts/unstable/genesis/net/hosts

@@ -35,14 +35,14 @@
 # Tailscale hosts
 100.119.228.115 nas.home chronicles.shire-zebra.ts.net
 100.88.91.27 dns.home
-100.80.99.48 exodus.home
+100.70.99.91 exodus.home exodus.shire-zebra.ts.net
-100.96.198.104 genesis.home smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home
+100.96.198.104 genesis.home genesis.shire-zebra.ts.net smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home
 100.117.28.111 gitlab.home gitlab.shire-zebra.ts.net gitlab.thehellings.lan registry.thehellings.lan git.thehellings.lan
 100.68.203.1 hosea.home hosea.shire-zebra.ts.net grafana.home
 100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes longhorn.kubernetes
 100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes immich.kubernetes postgres.kubernetes
-100.90.74.19 zeke.home
+100.90.74.19 zeke.home zeke.shire-zebra.ts.net
-100.115.57.8 linode.home
+100.109.86.8 linode.home linode.shire-zebra.ts.net
 100.65.5.38 matrix.home matrix.shire-zebra.ts.net
 #100.127.55.22 jellyfin.home
 100.114.187.61 nas1.home nas1.shire-zebra.ts.net

hosts/unstable/genesis/networking.nix

@@ -9,7 +9,7 @@ let
   lanIP = metadata.hosts.${config.networking.hostName}.ip;
   iot = "enp2s0";
   iotIP = "192.168.66.250";
-  routerIP = metadata.infra.gw;
+  #routerIP = metadata.infra.gw;
   extraHosts = builtins.readFile ./net/hosts;
 
   proxyPort = 3128;
@@ -81,19 +81,6 @@ in
   environment.etc."hosts.d/local".text = extraHosts;
 
   services = {
-    kea = {
-      dhcp4 = (
-        import ./networking/dhcp.nix {
-          inherit
-            iot
-            lan
-            lanIP
-            routerIP
-            ;
-        }
-      );
-    };
-
     #########
     # dnsmasq config
     ########

hosts/unstable/hosea/default.nix

@@ -285,7 +285,7 @@ in
         "uid": "kubernetes-overview",
         "title": "Kubernetes Overview",
         "schemaVersion": 38,
-        "version": 1,
+        "version": 2,
         "refresh": "30s",
         "time": {"from": "now-3h", "to": "now"},
         "panels": [
@@ -294,14 +294,14 @@ in
             "type": "stat",
             "title": "Running Pods",
             "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
-            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Running\"}) or vector(0)", "refId": "A"}]
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Running\"} == 1) or vector(0)", "refId": "A"}]
           },
           {
             "id": 2,
             "type": "stat",
             "title": "Failed Pods",
             "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
-            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Failed\"}) or vector(0)", "refId": "A"}],
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Failed\"} == 1) or vector(0)", "refId": "A"}],
             "fieldConfig": {
               "defaults": {
                 "thresholds": {
@@ -316,7 +316,7 @@ in
             "type": "stat",
             "title": "Pending Pods",
             "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4},
-            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Pending\"}) or vector(0)", "refId": "A"}],
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Pending\"} == 1) or vector(0)", "refId": "A"}],
             "fieldConfig": {
               "defaults": {
                 "thresholds": {
@@ -326,6 +326,21 @@ in
               }
             }
           },
+          {
+            "id": 8,
+            "type": "stat",
+            "title": "Nodes Ready",
+            "gridPos": {"x": 18, "y": 0, "w": 6, "h": 4},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_node_status_condition{condition=\"Ready\",status=\"true\"} == 1) or vector(0)", "refId": "A"}],
+            "fieldConfig": {
+              "defaults": {
+                "thresholds": {
+                  "mode": "absolute",
+                  "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]
+                }
+              }
+            }
+          },
           {
             "id": 4,
             "type": "timeseries",
@@ -333,25 +348,47 @@ in
             "gridPos": {"x": 0, "y": 4, "w": 24, "h": 8},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "topk(10, rate(kube_pod_container_status_restarts_total[15m]) * 900)", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}]
           },
+          {
+            "id": 9,
+            "type": "timeseries",
+            "title": "Container CPU Usage (cores)",
+            "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "topk(10, rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\"}[5m]))", "legendFormat": "{{namespace}}/{{pod}}/{{container}}", "refId": "A"}]
+          },
+          {
+            "id": 10,
+            "type": "timeseries",
+            "title": "Container Memory Usage",
+            "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "topk(10, container_memory_working_set_bytes{container!=\"\",container!=\"POD\"})", "legendFormat": "{{namespace}}/{{pod}}/{{container}}", "refId": "A"}],
+            "fieldConfig": {"defaults": {"unit": "bytes"}}
+          },
           {
             "id": 5,
             "type": "table",
             "title": "All Pods",
-            "gridPos": {"x": 0, "y": 12, "w": 24, "h": 10},
+            "gridPos": {"x": 0, "y": 20, "w": 24, "h": 10},
-            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kube_pod_status_phase", "instant": true, "refId": "A"}]
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kube_pod_status_phase == 1", "instant": true, "refId": "A"}]
+          },
+          {
+            "id": 11,
+            "type": "table",
+            "title": "Deployments",
+            "gridPos": {"x": 0, "y": 30, "w": 24, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kube_deployment_status_replicas_available", "instant": true, "refId": "A"}]
           },
           {
             "id": 6,
             "type": "timeseries",
             "title": "Node CPU Usage",
-            "gridPos": {"x": 0, "y": 22, "w": 12, "h": 8},
+            "gridPos": {"x": 0, "y": 38, "w": 12, "h": 8},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(node) (rate(node_cpu_seconds_total{mode=\"idle\",job=\"node\"}[5m])) * 100)", "refId": "A"}]
           },
           {
             "id": 7,
             "type": "timeseries",
             "title": "Node Memory Usage %",
-            "gridPos": {"x": 12, "y": 22, "w": 12, "h": 8},
+            "gridPos": {"x": 12, "y": 38, "w": 12, "h": 8},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "refId": "A"}]
           }
         ]
@@ -363,7 +400,7 @@ in
         "uid": "network-overview",
         "title": "Network & UniFi",
         "schemaVersion": 38,
-        "version": 1,
+        "version": 2,
         "refresh": "30s",
         "time": {"from": "now-3h", "to": "now"},
         "panels": [
@@ -371,43 +408,82 @@ in
             "id": 1,
             "type": "stat",
             "title": "DNS Queries/s",
-            "gridPos": {"x": 0, "y": 0, "w": 8, "h": 4},
+            "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(dnsmasq_queries_total[5m]) or vector(0)", "refId": "A"}]
           },
-          {
-            "id": 2,
-            "type": "stat",
-            "title": "DHCP Leases",
-            "gridPos": {"x": 8, "y": 0, "w": 8, "h": 4},
-            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kea_dhcp4_addresses_assigned_total or vector(0)", "refId": "A"}]
-          },
           {
             "id": 3,
             "type": "stat",
             "title": "UniFi Devices",
-            "gridPos": {"x": 16, "y": 0, "w": 8, "h": 4},
+            "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(unifipoller_device_uptime_seconds) or vector(0)", "refId": "A"}]
           },
+          {
+            "id": 7,
+            "type": "stat",
+            "title": "WiFi Clients",
+            "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(unifipoller_client_wifi_tx_rate_bps) or vector(0)", "refId": "A"}]
+          },
+          {
+            "id": 8,
+            "type": "stat",
+            "title": "Wired Clients",
+            "gridPos": {"x": 18, "y": 0, "w": 6, "h": 4},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(unifipoller_client_wired_tx_rate_bps) or vector(0)", "refId": "A"}]
+          },
+          {
+            "id": 9,
+            "type": "timeseries",
+            "title": "WAN RX (bytes/s)",
+            "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_device_wan_receive_bytes_total[5m])", "legendFormat": "{{name}}", "refId": "A"}],
+            "fieldConfig": {"defaults": {"unit": "Bps"}}
+          },
+          {
+            "id": 10,
+            "type": "timeseries",
+            "title": "WAN TX (bytes/s)",
+            "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_device_wan_transmit_bytes_total[5m])", "legendFormat": "{{name}}", "refId": "A"}],
+            "fieldConfig": {"defaults": {"unit": "Bps"}}
+          },
           {
             "id": 4,
             "type": "timeseries",
             "title": "UniFi Port RX (bytes/s)",
-            "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
+            "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_port_receive_bytes_total[5m])", "legendFormat": "{{port_id}} {{name}}", "refId": "A"}]
           },
           {
             "id": 5,
             "type": "timeseries",
             "title": "UniFi Port TX (bytes/s)",
-            "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
+            "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_port_transmit_bytes_total[5m])", "legendFormat": "{{port_id}} {{name}}", "refId": "A"}]
           },
+          {
+            "id": 11,
+            "type": "timeseries",
+            "title": "Top Client Throughput (bytes/s)",
+            "gridPos": {"x": 0, "y": 20, "w": 24, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "topk(10, rate(unifipoller_client_receive_bytes_total[5m]) + rate(unifipoller_client_transmit_bytes_total[5m]))", "legendFormat": "{{name}} {{ip}}", "refId": "A"}],
+            "fieldConfig": {"defaults": {"unit": "Bps"}}
+          },
           {
             "id": 6,
             "type": "timeseries",
             "title": "Ping Latency (ms)",
-            "gridPos": {"x": 0, "y": 12, "w": 24, "h": 8},
+            "gridPos": {"x": 0, "y": 28, "w": 12, "h": 8},
             "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "probe_duration_seconds{job=\"ping\"} * 1000", "legendFormat": "{{instance}}", "refId": "A"}]
+          },
+          {
+            "id": 12,
+            "type": "table",
+            "title": "Device Status",
+            "gridPos": {"x": 12, "y": 28, "w": 12, "h": 8},
+            "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "unifipoller_device_uptime_seconds", "instant": true, "refId": "A"}],
+            "fieldConfig": {"defaults": {"unit": "s"}}
           }
         ]
       }

manifests/monitoring/config.yaml

@@ -66,10 +66,6 @@ data:
             target_label: instance
           - target_label: __address__
             replacement: "genesis.shire-zebra.ts.net:9115"
-      - job_name: kea
-        static_configs:
-          - targets:
-              - "genesis.shire-zebra.ts.net:9547"
       - job_name: dnsmasq
         static_configs:
           - targets:
@@ -79,8 +75,14 @@ data:
         static_configs:
           - targets:
               - "hosea.shire-zebra.ts.net:9108"
-      # Restic rest-server backup metrics
+      # Restic rest-server backup metrics (HTTPS with basic auth)
       - job_name: restic_backups
+        scheme: https
+        tls_config:
+          insecure_skip_verify: true
+        basic_auth:
+          username_file: /etc/prometheus/secrets/restic/username
+          password_file: /etc/prometheus/secrets/restic/password
         static_configs:
           - targets:
               - "nas1.shire-zebra.ts.net:30248"
@@ -95,6 +97,11 @@ data:
         static_configs:
           - targets:
               - "unpoller.monitoring.svc.cluster.local:9130"
+      # kube-state-metrics
+      - job_name: kube_state_metrics
+        static_configs:
+          - targets:
+              - "kube-state-metrics.monitoring.svc.cluster.local:8080"
       # This scrapes metrics from the Kubernetes kubelets
       - job_name: kubelet
         kubernetes_sd_configs:
@@ -113,8 +120,6 @@ data:
           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
         metrics_path: /metrics/cadvisor
         bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-        #authorization:
-        #  credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       # This scrapes Service endpoints in Kubernetes
       - job_name: 'k8sservices'
         kubernetes_sd_configs:

manifests/monitoring/deployment.yaml

@@ -41,6 +41,9 @@ spec:
               subPath: alerts.yml
             - name: prometheus-storage-volume
               mountPath: /prometheus
+            - name: restic-credentials
+              mountPath: /etc/prometheus/secrets/restic
+              readOnly: true
       restartPolicy: Always
       volumes:
         - name: prometheus-config-values
@@ -54,3 +57,6 @@ spec:
         - name: prometheus-storage-volume
           persistentVolumeClaim:
             claimName: prometheus-data
+        - name: restic-credentials
+          secret:
+            secretName: restic-credentials

manifests/monitoring/kube-state-metrics.yaml

@@ -0,0 +1,102 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: "2.15.0"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: "2.15.0"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: monitoring
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: "2.15.0"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kube-state-metrics
+        app.kubernetes.io/version: "2.15.0"
+    spec:
+      serviceAccountName: kube-state-metrics
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        fsGroup: 65534
+      containers:
+        - name: kube-state-metrics
+          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.15.0
+          ports:
+            - name: metrics
+              containerPort: 8080
+            - name: telemetry
+              containerPort: 8081
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8081
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/version: "2.15.0"
+spec:
+  selector:
+    app.kubernetes.io/name: kube-state-metrics
+  ports:
+    - name: metrics
+      port: 8080
+      targetPort: metrics
+    - name: telemetry
+      port: 8081
+      targetPort: telemetry
+  clusterIP: None

manifests/monitoring/kustomization.yaml

@@ -10,3 +10,5 @@ resources:
   - service.yaml
   - ingress.yaml
   - unpoller.yaml
+  - kube-state-metrics.yaml
+  - restic-secret.yaml

manifests/monitoring/restic-secret.yaml

@@ -0,0 +1,22 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: restic-credentials
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: bitwarden-login
+    kind: ClusterSecretStore
+  target:
+    name: restic-credentials
+    creationPolicy: Owner
+  data:
+    - secretKey: username
+      remoteRef:
+        key: cb04cb1b-1037-40d6-a187-b375015e7ef6
+        property: username
+    - secretKey: password
+      remoteRef:
+        key: cb04cb1b-1037-40d6-a187-b375015e7ef6
+        property: password

modules/nixos/kubernetes.nix

@@ -67,6 +67,9 @@ in
         443
         5432
         6443
+        8080
+        8443
+        10250
       ]
       ++ (
         if cfg.agentOnly then

network.json

@@ -15,7 +15,7 @@
     "exodus": {
       "ip": null,
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3",
-      "ts": "100.80.99.48",
+      "ts": "100.70.99.91",
       "system": "x86_64-linux"
     },
     "genesis": {