root
e76dfee1d8
Prometheus runs as a pod and only has Tailscale routes to the three k8s nodes it runs on (isaiah, jeremiah, zeke) plus hosea. Scraping genesis, exodus, and linode via *.shire-zebra.ts.net fails instantly (~1ms, connection refused) because those Tailscale routes don't exist inside the cluster network. Fix by host type: - k8s nodes (isaiah, jeremiah, zeke, hosea): keep Tailscale hostnames - LAN hosts (genesis): use LAN IP 10.42.1.5 - Roaming/VPS (exodus, linode): use Tailscale IPs directly (100.x.x.x) since they have no fixed LAN address but the k8s nodes do route to other Tailscale IPs via the subnet router Apply same logic to kea, dnsmasq (genesis-only jobs) and the systemd/ping exporter jobs.
159 lines
5.9 KiB
YAML
159 lines
5.9 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 10s
|
|
evaluation_interval: 10s
|
|
alerting:
|
|
alertmanagers: []
|
|
rule_files:
|
|
- /etc/prometheus/alerts.yml
|
|
scrape_configs:
|
|
# This scrapes endpoints on my servers
|
|
# k8s nodes (isaiah, jeremiah, zeke) are reachable via Tailscale from within the cluster.
|
|
# Non-k8s hosts are reached via LAN IPs since Prometheus pods only have Tailscale
|
|
# routes for the nodes they run on.
|
|
- job_name: node
|
|
static_configs:
|
|
- targets:
|
|
# k8s nodes - reachable via Tailscale from within cluster
|
|
- "isaiah.shire-zebra.ts.net:9100"
|
|
- "jeremiah.shire-zebra.ts.net:9100"
|
|
- "zeke.shire-zebra.ts.net:9100"
|
|
- "hosea.shire-zebra.ts.net:9100"
|
|
# non-k8s hosts - reached via LAN IP
|
|
- "10.42.1.5:9100" # genesis
|
|
- "10.42.1.14:9100" # nas1 (if node exporter added later)
|
|
- targets:
|
|
- "100.80.99.48:9100" # exodus (laptop, LAN IP varies - use Tailscale IP directly)
|
|
- "100.109.86.8:9100" # linode (VPS, no LAN IP - use Tailscale IP directly)
|
|
- job_name: systemd
|
|
static_configs:
|
|
- targets:
|
|
- "isaiah.shire-zebra.ts.net:9558"
|
|
- "jeremiah.shire-zebra.ts.net:9558"
|
|
- "zeke.shire-zebra.ts.net:9558"
|
|
- "hosea.shire-zebra.ts.net:9558"
|
|
- "10.42.1.5:9558" # genesis
|
|
- "100.80.99.48:9558" # exodus
|
|
- "100.109.86.8:9558" # linode
|
|
- job_name: ping
|
|
static_configs:
|
|
- targets:
|
|
- "isaiah.shire-zebra.ts.net:9427"
|
|
- "jeremiah.shire-zebra.ts.net:9427"
|
|
- "zeke.shire-zebra.ts.net:9427"
|
|
- "hosea.shire-zebra.ts.net:9427"
|
|
- "10.42.1.5:9427" # genesis
|
|
- "100.80.99.48:9427" # exodus
|
|
- "100.109.86.8:9427" # linode
|
|
# ICMP ping reachability for non-NixOS infrastructure
|
|
- job_name: infra_ping
|
|
metrics_path: /probe
|
|
params:
|
|
module: [icmp]
|
|
static_configs:
|
|
- targets:
|
|
- "10.42.0.3" # OpenWRT access point
|
|
- "10.42.0.4" # Joel (Proxmox)
|
|
- "10.42.1.1" # pve1 (Proxmox)
|
|
- "10.42.1.4" # chronicles (Synology NAS)
|
|
- "10.42.1.14" # nas1 (TrueNAS)
|
|
- "10.42.2.57" # odoo
|
|
- "10.42.2.89" # mattermost
|
|
- "10.42.2.155" # homeassistant
|
|
- "10.42.3.58" # USW-Pro-HD-24 (UniFi switch)
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: "genesis.shire-zebra.ts.net:9115"
|
|
- job_name: kea
|
|
static_configs:
|
|
- targets:
|
|
- "10.42.1.5:9547" # genesis (LAN IP - not a k8s node)
|
|
- job_name: dnsmasq
|
|
static_configs:
|
|
- targets:
|
|
- "10.42.1.5:9153" # genesis (LAN IP - not a k8s node)
|
|
# Converter from TrueNAS
|
|
- job_name: graphite
|
|
static_configs:
|
|
- targets:
|
|
- "hosea.shire-zebra.ts.net:9108"
|
|
# Restic rest-server backup metrics
|
|
- job_name: restic_backups
|
|
static_configs:
|
|
- targets:
|
|
- "nas1.shire-zebra.ts.net:30248"
|
|
# Mattermost metrics
|
|
- job_name: mattermost
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets:
|
|
- "mattermost.thehellings.lan:8065"
|
|
# UniFi metrics via unpoller
|
|
- job_name: unifi
|
|
static_configs:
|
|
- targets:
|
|
- "unpoller.monitoring.svc.cluster.local:9130"
|
|
# This scrapes metrics from the Kubernetes kubelets
|
|
- job_name: kubelet
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
# This scrapes metrics about the containers themselves
|
|
- job_name: cadvisor
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
metrics_path: /metrics/cadvisor
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
#authorization:
|
|
# credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
# This scrapes Service endpoints in Kubernetes
|
|
- job_name: 'k8sservices'
|
|
kubernetes_sd_configs:
|
|
- role: endpointslice
|
|
relabel_configs:
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_service_name
|
|
action: drop
|
|
regex: "default;kubernetes"
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
regex: default
|
|
action: keep
|
|
- source_labels:
|
|
- __meta_kubernetes_service_name
|
|
target_label: job
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
# This scrapes any exposed pod port named "metrics", assuming that it is
|
|
# a prometheus source
|
|
- job_name: k8spods
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
relabel_configs:
|
|
- source_labels:
|
|
- __meta_kubernetes_pod_container_port_name
|
|
regex: metrics
|
|
action: keep
|
|
- source_labels:
|
|
- __meta_kubernetes_pod_container_name
|
|
target_label: job
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|