Add Prometheus scraping to Kubernetes

manifests/kustomization.yaml

@@ -8,3 +8,4 @@ resources:
   - matrix
   - gitlab-runner
   - immich
+  - monitoring

manifests/monitoring/cluster-role.yaml

@@ -0,0 +1,41 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - services
+      - endpoints
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+  - kind: ServiceAccount
+    name: prometheus
+    namespace: monitoring

manifests/monitoring/config.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 10s
+      evaluation_interval: 10s
+    alerting:
+    scrape_configs:
+      # This scrapes endpoints on my servers
+      - job_name: prometheus
+        static_configs:
+          - targets:
+              # Node exporters
+              - "genesis.shire-zebra.ts.net:9100"
+              - "isaiah.shire-zebra.ts.net:9100"
+              - "jeremiah.shire-zebra.ts.net:9100"
+              - "zeke.shire-zebra.ts.net:9100"
+              - "hosea.shire-zebra.ts.net:9100"
+              - "exodus.shire-zebra.ts.net:9100"
+              - "linode.shire-zebra.ts.net:9100"
+              - "vm-gitlab.shire-zebra.ts.net:9100"
+              # Systemd exporters
+              - "genesis.shire-zebra.ts.net:9558"
+              - "isaiah.shire-zebra.ts.net:9558"
+              - "jeremiah.shire-zebra.ts.net:9558"
+              - "zeke.shire-zebra.ts.net:9558"
+              - "hosea.shire-zebra.ts.net:9558"
+              - "exodus.shire-zebra.ts.net:9558"
+              - "linode.shire-zebra.ts.net:9558"
+              - "vm-gitlab.shire-zebra.ts.net:9558"
+              # Ping port
+              - "genesis.shire-zebra.ts.net:9427"
+              - "isaiah.shire-zebra.ts.net:9427"
+              - "jeremiah.shire-zebra.ts.net:9427"
+              - "zeke.shire-zebra.ts.net:9427"
+              - "hosea.shire-zebra.ts.net:9427"
+              - "exodus.shire-zebra.ts.net:9427"
+              - "linode.shire-zebra.ts.net:9427"
+              - "vm-gitlab.shire-zebra.ts.net:9427"
+              # Key port
+              - "genesis.shire-zebra.ts.net:9547"
+              # Dnsmasq port
+              - "genesis.shire-zebra.ts.net:9153"
+              # Graphite exporter
+              - "hosea.shire-zebra.ts.net:9108"
+      # This scrapes metrics from the Kubernetes kubelets
+      #- job_name: kubelet
+        #kubernetes_sd_configs:
+          #- role: node
+        #scheme: https
+        #tls_config:
+          #ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          #insecure_skip_verify: true
+      ## This scrapes metrics about the containers themselves
+      #- job_name: cadvisor
+        #kubernetes_sd_configs:
+          #- role: node
+        #scheme: https
+        #tls_config:
+          #ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          #insecure_skip_verify: true
+        #metrics_path: /metrics/cadvisor
+      ## This scrapes Service endpoints in Kubernetes
+      #- job_name: 'k8sservices'
+        #kubernetes_sd_configs:
+          #- role: endpoints
+        #relabel_configs:
+          #- source_labels:
+              #- __meta_kubernetes_namespace
+              #- __meta_kubernetes_service_name
+            #action: drop
+            #regex: "default;kubernetes"
+          #- source_labels:
+              #- __meta_kubernetes_namespace
+            #regex: default
+            #action: keep
+          #- source_labels:
+              #- __meta_kubernetes_service_name
+            #target_label: job
+      ## This scrapes any exposed pod port named "metrics", assuming that it is
+      ## a prometheus source
+      #- job_name: k8spods
+        #kubernetes_sd_configs:
+          #- role: pod
+        #relabel_configs:
+          #- source_labels:
+              #- __meta_kubernetes_pod_container_port_name
+            #regex: metrics
+            #action: keep
+          #- source_labels:
+              #- __meta_kubernetes_pod_container_name
+            #target_label: job

manifests/monitoring/deployment.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9090"
+    spec:
+      containers:
+        - name: prometheus
+          image: prom/prometheus
+          args:
+            - "--storage.tsdb.path=/prometheus"
+            - "--config.file=/etc/prometheus/prometheus.yml"
+          ports:
+            - name: http
+              containerPort: 9090
+          volumeMounts:
+            - name: prometheus-config-values
+              mountPath: /etc/prometheus
+            - name: prometheus-storage-volume
+              mountPath: /prometheus
+      restartPolicy: Always
+      volumes:
+        - name: prometheus-config-values
+          configMap:
+            defaultMode: 420
+            name: prometheus-config
+        - name: prometheus-storage-volume
+          persistentVolumeClaim:
+            claimName: prometheus-data

manifests/monitoring/ingress.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus-tailscale
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: prometheus
+      port:
+        name: http
+  tls:
+    - hosts:
+        - prometheus

manifests/monitoring/kustomization.yaml

@@ -0,0 +1,10 @@
+namespace: monitoring
+
+resources:
+  - namespace.yaml
+  - pvc.yaml
+  - cluster-role.yaml
+  - config.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

manifests/monitoring/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

manifests/monitoring/pvc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: prometheus-data
+  labels:
+    app: prometheus-deployment
+spec:
+  storageClassName: longhorn-default
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: "20Gi"

manifests/monitoring/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  labels:
+    app: prometheus
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9090"
+spec:
+  selector:
+    app: prometheus
+  type: ClusterIP
+  ports:
+    - port: 9090
+      targetPort: http
+      protocol: TCP
+      name: http