From 82acff58570de3db9f45ffb1b8a575a724472411 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 22 Nov 2025 20:14:55 -0600 Subject: [PATCH] Add Prometheus scraping to Kubernetes --- manifests/kustomization.yaml | 1 + manifests/monitoring/cluster-role.yaml | 41 +++++++++++ manifests/monitoring/config.yaml | 95 +++++++++++++++++++++++++ manifests/monitoring/deployment.yaml | 47 ++++++++++++ manifests/monitoring/ingress.yaml | 14 ++++ manifests/monitoring/kustomization.yaml | 10 +++ manifests/monitoring/namespace.yaml | 4 ++ manifests/monitoring/pvc.yaml | 13 ++++ manifests/monitoring/service.yaml | 18 +++++ modules/hm/gui/bookmarks.nix | 2 +- 10 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 manifests/monitoring/cluster-role.yaml create mode 100644 manifests/monitoring/config.yaml create mode 100644 manifests/monitoring/deployment.yaml create mode 100644 manifests/monitoring/ingress.yaml create mode 100644 manifests/monitoring/kustomization.yaml create mode 100644 manifests/monitoring/namespace.yaml create mode 100644 manifests/monitoring/pvc.yaml create mode 100644 manifests/monitoring/service.yaml diff --git a/manifests/kustomization.yaml b/manifests/kustomization.yaml index 2973be6..b969829 100644 --- a/manifests/kustomization.yaml +++ b/manifests/kustomization.yaml @@ -8,3 +8,4 @@ resources: - matrix - gitlab-runner - immich + - monitoring diff --git a/manifests/monitoring/cluster-role.yaml b/manifests/monitoring/cluster-role.yaml new file mode 100644 index 0000000..56a3bc8 --- /dev/null +++ b/manifests/monitoring/cluster-role.yaml @@ -0,0 +1,41 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring diff --git a/manifests/monitoring/config.yaml b/manifests/monitoring/config.yaml new file mode 100644 index 0000000..9cc8580 --- /dev/null +++ b/manifests/monitoring/config.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config +data: + prometheus.yml: | + global: + scrape_interval: 10s + evaluation_interval: 10s + alerting: + scrape_configs: + # This scrapes endpoints on my servers + - job_name: prometheus + static_configs: + - targets: + # Node exporters + - "genesis.shire-zebra.ts.net:9100" + - "isaiah.shire-zebra.ts.net:9100" + - "jeremiah.shire-zebra.ts.net:9100" + - "zeke.shire-zebra.ts.net:9100" + - "hosea.shire-zebra.ts.net:9100" + - "exodus.shire-zebra.ts.net:9100" + - "linode.shire-zebra.ts.net:9100" + - "vm-gitlab.shire-zebra.ts.net:9100" + # Systemd exporters + - "genesis.shire-zebra.ts.net:9558" + - "isaiah.shire-zebra.ts.net:9558" + - "jeremiah.shire-zebra.ts.net:9558" + - "zeke.shire-zebra.ts.net:9558" + - "hosea.shire-zebra.ts.net:9558" + - "exodus.shire-zebra.ts.net:9558" + - "linode.shire-zebra.ts.net:9558" + - "vm-gitlab.shire-zebra.ts.net:9558" + # Ping port + - "genesis.shire-zebra.ts.net:9427" + - "isaiah.shire-zebra.ts.net:9427" + - "jeremiah.shire-zebra.ts.net:9427" + - "zeke.shire-zebra.ts.net:9427" + - "hosea.shire-zebra.ts.net:9427" + - "exodus.shire-zebra.ts.net:9427" + - "linode.shire-zebra.ts.net:9427" + - "vm-gitlab.shire-zebra.ts.net:9427" + # Key port + - "genesis.shire-zebra.ts.net:9547" + # Dnsmasq port + - "genesis.shire-zebra.ts.net:9153" + # Graphite exporter + - "hosea.shire-zebra.ts.net:9108" + # This scrapes metrics from the Kubernetes kubelets + #- job_name: kubelet + #kubernetes_sd_configs: + #- role: node + #scheme: https + #tls_config: + #ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + #insecure_skip_verify: true + ## This scrapes metrics about the containers themselves + #- job_name: cadvisor + #kubernetes_sd_configs: + #- role: node + #scheme: https + #tls_config: + #ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + #insecure_skip_verify: true + #metrics_path: /metrics/cadvisor + ## This scrapes Service endpoints in Kubernetes + #- job_name: 'k8sservices' + #kubernetes_sd_configs: + #- role: endpoints + #relabel_configs: + #- source_labels: + #- __meta_kubernetes_namespace + #- __meta_kubernetes_service_name + #action: drop + #regex: "default;kubernetes" + #- source_labels: + #- __meta_kubernetes_namespace + #regex: default + #action: keep + #- source_labels: + #- __meta_kubernetes_service_name + #target_label: job + ## This scrapes any exposed pod port named "metrics", assuming that it is + ## a prometheus source + #- job_name: k8spods + #kubernetes_sd_configs: + #- role: pod + #relabel_configs: + #- source_labels: + #- __meta_kubernetes_pod_container_port_name + #regex: metrics + #action: keep + #- source_labels: + #- __meta_kubernetes_pod_container_name + #target_label: job diff --git a/manifests/monitoring/deployment.yaml b/manifests/monitoring/deployment.yaml new file mode 100644 index 0000000..ba4aa14 --- /dev/null +++ b/manifests/monitoring/deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + labels: + app: prometheus +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + containers: + - name: prometheus + image: prom/prometheus + args: + - "--storage.tsdb.path=/prometheus" + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - name: http + containerPort: 9090 + volumeMounts: + - name: prometheus-config-values + mountPath: /etc/prometheus + - name: prometheus-storage-volume + mountPath: /prometheus + restartPolicy: Always + volumes: + - name: prometheus-config-values + configMap: + defaultMode: 420 + name: prometheus-config + - name: prometheus-storage-volume + persistentVolumeClaim: + claimName: prometheus-data diff --git a/manifests/monitoring/ingress.yaml b/manifests/monitoring/ingress.yaml new file mode 100644 index 0000000..70c4d48 --- /dev/null +++ b/manifests/monitoring/ingress.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-tailscale +spec: + ingressClassName: tailscale + defaultBackend: + service: + name: prometheus + port: + name: http + tls: + - hosts: + - prometheus diff --git a/manifests/monitoring/kustomization.yaml b/manifests/monitoring/kustomization.yaml new file mode 100644 index 0000000..cff2df8 --- /dev/null +++ b/manifests/monitoring/kustomization.yaml @@ -0,0 +1,10 @@ +namespace: monitoring + +resources: + - namespace.yaml + - pvc.yaml + - cluster-role.yaml + - config.yaml + - deployment.yaml + - service.yaml + - ingress.yaml diff --git a/manifests/monitoring/namespace.yaml b/manifests/monitoring/namespace.yaml new file mode 100644 index 0000000..d325236 --- /dev/null +++ b/manifests/monitoring/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring diff --git a/manifests/monitoring/pvc.yaml b/manifests/monitoring/pvc.yaml new file mode 100644 index 0000000..b409f03 --- /dev/null +++ b/manifests/monitoring/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-data + labels: + app: prometheus-deployment +spec: + storageClassName: longhorn-default + accessModes: + - ReadWriteMany + resources: + requests: + storage: "20Gi" diff --git a/manifests/monitoring/service.yaml b/manifests/monitoring/service.yaml new file mode 100644 index 0000000..ccaf75e --- /dev/null +++ b/manifests/monitoring/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + labels: + app: prometheus + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" +spec: + selector: + app: prometheus + type: ClusterIP + ports: + - port: 9090 + targetPort: http + protocol: TCP + name: http diff --git a/modules/hm/gui/bookmarks.nix b/modules/hm/gui/bookmarks.nix index 1f0803c..3bd6f0d 100644 --- a/modules/hm/gui/bookmarks.nix +++ b/modules/hm/gui/bookmarks.nix @@ -273,7 +273,7 @@ } { name = "Prometheus"; - url = "http://hosea.shire-zebra.ts.net:9090/"; + url = "http://prometheus.shire-zebra.ts.net/"; } { name = "Grafana";