Enable node level monitoring

manifests/monitoring/cluster-role.yaml

@@ -1,24 +1,135 @@
+# This doesn't seem to make enough of the things
+# visible, still, and will still generate
+# a 403 error when hitting the kubelet and
+# cadvisor endpoints, so we use
+# cluster-admin down farther
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: prometheus
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 2.17.0
+  name: kube-state-metrics
 rules:
-  - apiGroups: [""]
+  - apiGroups:
+      - ""
     resources:
+      - configmaps
+      - secrets
       - nodes
-      - services
-      - endpoints
       - pods
+      - services
+      - serviceaccounts
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - namespaces
+      - endpoints
     verbs:
-      - get
       - list
       - watch
   - apiGroups:
-      - extensions
+      - apps
     resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+      - jobs
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - certificatesigningrequests
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+      - volumeattachments
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - mutatingwebhookconfigurations
+      - validatingwebhookconfigurations
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+      - ingressclasses
       - ingresses
     verbs:
-      - get
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterrolebindings
+      - clusterroles
+      - rolebindings
+      - roles
+    verbs:
       - list
       - watch
 ---
@@ -34,7 +145,8 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: prometheus
+  name: cluster-admin
+  #name: prometheus
 subjects:
   - kind: ServiceAccount
     name: prometheus

manifests/monitoring/config.yaml

@@ -73,12 +73,13 @@ data:
         tls_config:
           ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
         metrics_path: /metrics/cadvisor
-        authorization:
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        #authorization:
+        #  credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
       # This scrapes Service endpoints in Kubernetes
       - job_name: 'k8sservices'
         kubernetes_sd_configs:
-          - role: endpoints
+          - role: endpointslice
         relabel_configs:
           - source_labels:
               - __meta_kubernetes_namespace

modules/nixos/kubernetes.nix

@@ -61,6 +61,7 @@ in
         443
         5432
         6443
+        10250
       ]
       ++ (
         if cfg.agentOnly then
@@ -115,6 +116,7 @@ in
           "--write-kubeconfig-group kubeconfig"
           "--resolv-conf=/etc/resolv.conf"
           "--node-label node.longhorn.io/create-default-disk=config"
+          "--supervisor-metrics=true"
           "--tls-san ${config.networking.hostName}.home"
           "--tls-san ${config.networking.hostName}.thehellings.lan"
           "--tls-san ${config.networking.hostName}.shire-zebra.ts.net"