From e76dfee1d8864f245e95011a1ea2c137b4718553 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 25 Mar 2026 20:58:34 -0500 Subject: [PATCH] fix: use LAN IPs for non-k8s hosts in Prometheus scrape config Prometheus runs as a pod and only has Tailscale routes to the three k8s nodes it runs on (isaiah, jeremiah, zeke) plus hosea. Scraping genesis, exodus, and linode via *.shire-zebra.ts.net fails instantly (~1ms, connection refused) because those Tailscale routes don't exist inside the cluster network. Fix by host type: - k8s nodes (isaiah, jeremiah, zeke, hosea): keep Tailscale hostnames - LAN hosts (genesis): use LAN IP 10.42.1.5 - Roaming/VPS (exodus, linode): use Tailscale IPs directly (100.x.x.x) since they have no fixed LAN address but the k8s nodes do route to other Tailscale IPs via the subnet router Apply same logic to kea, dnsmasq (genesis-only jobs) and the systemd/ping exporter jobs. --- manifests/monitoring/config.yaml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/manifests/monitoring/config.yaml b/manifests/monitoring/config.yaml index 471d317..03733a2 100644 --- a/manifests/monitoring/config.yaml +++ b/manifests/monitoring/config.yaml @@ -13,36 +13,43 @@ data: - /etc/prometheus/alerts.yml scrape_configs: # This scrapes endpoints on my servers + # k8s nodes (isaiah, jeremiah, zeke) are reachable via Tailscale from within the cluster. + # Non-k8s hosts are reached via LAN IPs since Prometheus pods only have Tailscale + # routes for the nodes they run on. - job_name: node static_configs: - targets: - - "genesis.shire-zebra.ts.net:9100" + # k8s nodes - reachable via Tailscale from within cluster - "isaiah.shire-zebra.ts.net:9100" - "jeremiah.shire-zebra.ts.net:9100" - "zeke.shire-zebra.ts.net:9100" - "hosea.shire-zebra.ts.net:9100" - - "exodus.shire-zebra.ts.net:9100" - - "linode.shire-zebra.ts.net:9100" + # non-k8s hosts - reached via LAN IP + - "10.42.1.5:9100" # genesis + - "10.42.1.14:9100" # nas1 (if node exporter added later) + - targets: + - "100.80.99.48:9100" # exodus (laptop, LAN IP varies - use Tailscale IP directly) + - "100.109.86.8:9100" # linode (VPS, no LAN IP - use Tailscale IP directly) - job_name: systemd static_configs: - targets: - - "genesis.shire-zebra.ts.net:9558" - "isaiah.shire-zebra.ts.net:9558" - "jeremiah.shire-zebra.ts.net:9558" - "zeke.shire-zebra.ts.net:9558" - "hosea.shire-zebra.ts.net:9558" - - "exodus.shire-zebra.ts.net:9558" - - "linode.shire-zebra.ts.net:9558" + - "10.42.1.5:9558" # genesis + - "100.80.99.48:9558" # exodus + - "100.109.86.8:9558" # linode - job_name: ping static_configs: - targets: - - "genesis.shire-zebra.ts.net:9427" - "isaiah.shire-zebra.ts.net:9427" - "jeremiah.shire-zebra.ts.net:9427" - "zeke.shire-zebra.ts.net:9427" - "hosea.shire-zebra.ts.net:9427" - - "exodus.shire-zebra.ts.net:9427" - - "linode.shire-zebra.ts.net:9427" + - "10.42.1.5:9427" # genesis + - "100.80.99.48:9427" # exodus + - "100.109.86.8:9427" # linode # ICMP ping reachability for non-NixOS infrastructure - job_name: infra_ping metrics_path: /probe @@ -69,11 +76,11 @@ data: - job_name: kea static_configs: - targets: - - "genesis.shire-zebra.ts.net:9547" + - "10.42.1.5:9547" # genesis (LAN IP - not a k8s node) - job_name: dnsmasq static_configs: - targets: - - "genesis.shire-zebra.ts.net:9153" + - "10.42.1.5:9153" # genesis (LAN IP - not a k8s node) # Converter from TrueNAS - job_name: graphite static_configs: