nixos/manifests/monitoring/config.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
data:
  prometheus.yml: |
    global:
      scrape_interval: 10s
      evaluation_interval: 10s
    alerting:
      alertmanagers: []
    rule_files:
      - /etc/prometheus/alerts.yml
    scrape_configs:
      # This scrapes endpoints on my servers
      # k8s nodes (isaiah, jeremiah, zeke) are reachable via Tailscale from within the cluster.
      # Non-k8s hosts are reached via LAN IPs since Prometheus pods only have Tailscale
      # routes for the nodes they run on.
      - job_name: node
        static_configs:
          - targets:
              # k8s nodes - reachable via Tailscale from within cluster
              - "isaiah.shire-zebra.ts.net:9100"
              - "jeremiah.shire-zebra.ts.net:9100"
              - "zeke.shire-zebra.ts.net:9100"
              - "hosea.shire-zebra.ts.net:9100"
              # non-k8s hosts - reached via LAN IP
              - "10.42.1.5:9100"   # genesis
              - "10.42.1.14:9100"  # nas1 (if node exporter added later)
          - targets:
              - "100.80.99.48:9100"  # exodus (laptop, LAN IP varies - use Tailscale IP directly)
              - "100.109.86.8:9100"  # linode (VPS, no LAN IP - use Tailscale IP directly)
      - job_name: systemd
        static_configs:
          - targets:
              - "isaiah.shire-zebra.ts.net:9558"
              - "jeremiah.shire-zebra.ts.net:9558"
              - "zeke.shire-zebra.ts.net:9558"
              - "hosea.shire-zebra.ts.net:9558"
              - "10.42.1.5:9558"     # genesis
              - "100.80.99.48:9558"  # exodus
              - "100.109.86.8:9558"  # linode
      - job_name: ping
        static_configs:
          - targets:
              - "isaiah.shire-zebra.ts.net:9427"
              - "jeremiah.shire-zebra.ts.net:9427"
              - "zeke.shire-zebra.ts.net:9427"
              - "hosea.shire-zebra.ts.net:9427"
              - "10.42.1.5:9427"     # genesis
              - "100.80.99.48:9427"  # exodus
              - "100.109.86.8:9427"  # linode
      # ICMP ping reachability for non-NixOS infrastructure
      - job_name: infra_ping
        metrics_path: /probe
        params:
          module: [icmp]
        static_configs:
          - targets:
              - "10.42.0.3"    # OpenWRT access point
              - "10.42.0.4"    # Joel (Proxmox)
              - "10.42.1.1"    # pve1 (Proxmox)
              - "10.42.1.4"    # chronicles (Synology NAS)
              - "10.42.1.14"   # nas1 (TrueNAS)
              - "10.42.2.57"   # odoo
              - "10.42.2.89"   # mattermost
              - "10.42.2.155"  # homeassistant
              - "10.42.3.58"   # USW-Pro-HD-24 (UniFi switch)
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: "genesis.shire-zebra.ts.net:9115"
      - job_name: kea
        static_configs:
          - targets:
              - "10.42.1.5:9547"   # genesis (LAN IP - not a k8s node)
      - job_name: dnsmasq
        static_configs:
          - targets:
              - "10.42.1.5:9153"   # genesis (LAN IP - not a k8s node)
      # Converter from TrueNAS
      - job_name: graphite
        static_configs:
          - targets:
              - "hosea.shire-zebra.ts.net:9108"
      # Restic rest-server backup metrics
      - job_name: restic_backups
        static_configs:
          - targets:
              - "nas1.shire-zebra.ts.net:30248"
      # Mattermost metrics
      - job_name: mattermost
        metrics_path: /metrics
        static_configs:
          - targets:
              - "mattermost.thehellings.lan:8065"
      # UniFi metrics via unpoller
      - job_name: unifi
        static_configs:
          - targets:
              - "unpoller.monitoring.svc.cluster.local:9130"
      # This scrapes metrics from the Kubernetes kubelets
      - job_name: kubelet
        kubernetes_sd_configs:
          - role: node
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        authorization:
          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      # This scrapes metrics about the containers themselves
      - job_name: cadvisor
        kubernetes_sd_configs:
          - role: node
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        metrics_path: /metrics/cadvisor
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        #authorization:
        #  credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      # This scrapes Service endpoints in Kubernetes
      - job_name: 'k8sservices'
        kubernetes_sd_configs:
          - role: endpointslice
        relabel_configs:
          - source_labels:
              - __meta_kubernetes_namespace
              - __meta_kubernetes_service_name
            action: drop
            regex: "default;kubernetes"
          - source_labels:
              - __meta_kubernetes_namespace
            regex: default
            action: keep
          - source_labels:
              - __meta_kubernetes_service_name
            target_label: job
        authorization:
          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      # This scrapes any exposed pod port named "metrics", assuming that it is
      # a prometheus source
      - job_name: k8spods
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels:
              - __meta_kubernetes_pod_container_port_name
            regex: metrics
            action: keep
          - source_labels:
              - __meta_kubernetes_pod_container_name
            target_label: job
        authorization:
          credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token