root
7fee882488
buildbot/nix-eval Build done.
Issue 1 - Grafana secret key:
- Replace hardcoded '123456789' secret_key in hosea Grafana config
with an agenix-managed secret ($__file interpolation)
- Add age.secrets.grafana-secret-key declaration to hosea config
- Register grafana-secret-key.age in secrets/secrets.nix (publicKeys=everyone)
- NOTE: The .age file still needs to be created by running:
agenix -e secrets/grafana-secret-key.age
and entering a strong random secret (e.g. from 'pwgen -s 64 1')
Issue 2 - Prometheus stale targets:
- Remove retired vm-gitlab host from all three scrape jobs
(node, systemd, ping) in manifests/monitoring/config.yaml
- linode was already present in all scrape jobs
Issue 3 - Prometheus alerting gaps:
- Wire up rule_files in prometheus.yml pointing at alerts.yml
- Add manifests/monitoring/alerts.yaml with alerting rules:
* HostDown (node unreachable > 2m) - critical
* HighCPULoad (>90% for 10m) - warning
* LowDiskSpace (<10% free) - warning
* CriticalDiskSpace (<5% free) - critical
* HighMemoryUsage (>90% for 10m) - warning
* ResticBackupStaleness (no success > 26h) - warning
* PodCrashLooping - warning
* PodNotReady (>10m) - warning
- Mount alerts ConfigMap as separate volume subPath in deployment
so prometheus.yml and alerts.yml are both accessible at
/etc/prometheus/ without conflicts
113 lines
3.9 KiB
YAML
113 lines
3.9 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-config
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 10s
|
|
evaluation_interval: 10s
|
|
alerting:
|
|
alertmanagers: []
|
|
rule_files:
|
|
- /etc/prometheus/alerts.yml
|
|
scrape_configs:
|
|
# This scrapes endpoints on my servers
|
|
- job_name: node
|
|
static_configs:
|
|
- targets:
|
|
- "genesis.shire-zebra.ts.net:9100"
|
|
- "isaiah.shire-zebra.ts.net:9100"
|
|
- "jeremiah.shire-zebra.ts.net:9100"
|
|
- "zeke.shire-zebra.ts.net:9100"
|
|
- "hosea.shire-zebra.ts.net:9100"
|
|
- "exodus.shire-zebra.ts.net:9100"
|
|
- "linode.shire-zebra.ts.net:9100"
|
|
- job_name: systemd
|
|
static_configs:
|
|
- targets:
|
|
- "genesis.shire-zebra.ts.net:9558"
|
|
- "isaiah.shire-zebra.ts.net:9558"
|
|
- "jeremiah.shire-zebra.ts.net:9558"
|
|
- "zeke.shire-zebra.ts.net:9558"
|
|
- "hosea.shire-zebra.ts.net:9558"
|
|
- "exodus.shire-zebra.ts.net:9558"
|
|
- "linode.shire-zebra.ts.net:9558"
|
|
- job_name: ping
|
|
static_configs:
|
|
- targets:
|
|
- "genesis.shire-zebra.ts.net:9427"
|
|
- "isaiah.shire-zebra.ts.net:9427"
|
|
- "jeremiah.shire-zebra.ts.net:9427"
|
|
- "zeke.shire-zebra.ts.net:9427"
|
|
- "hosea.shire-zebra.ts.net:9427"
|
|
- "exodus.shire-zebra.ts.net:9427"
|
|
- "linode.shire-zebra.ts.net:9427"
|
|
- job_name: kea
|
|
static_configs:
|
|
- targets:
|
|
- "genesis.shire-zebra.ts.net:9547"
|
|
- job_name: dnsmasq
|
|
static_configs:
|
|
- targets:
|
|
- "genesis.shire-zebra.ts.net:9153"
|
|
# Converter from TrueNAS
|
|
- job_name: graphite
|
|
static_configs:
|
|
- targets:
|
|
- "hosea.shire-zebra.ts.net:9108"
|
|
# This scrapes metrics from the Kubernetes kubelets
|
|
- job_name: kubelet
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
# This scrapes metrics about the containers themselves
|
|
- job_name: cadvisor
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
scheme: https
|
|
tls_config:
|
|
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
metrics_path: /metrics/cadvisor
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
#authorization:
|
|
# credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
# This scrapes Service endpoints in Kubernetes
|
|
- job_name: 'k8sservices'
|
|
kubernetes_sd_configs:
|
|
- role: endpointslice
|
|
relabel_configs:
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
- __meta_kubernetes_service_name
|
|
action: drop
|
|
regex: "default;kubernetes"
|
|
- source_labels:
|
|
- __meta_kubernetes_namespace
|
|
regex: default
|
|
action: keep
|
|
- source_labels:
|
|
- __meta_kubernetes_service_name
|
|
target_label: job
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
# This scrapes any exposed pod port named "metrics", assuming that it is
|
|
# a prometheus source
|
|
- job_name: k8spods
|
|
kubernetes_sd_configs:
|
|
- role: pod
|
|
relabel_configs:
|
|
- source_labels:
|
|
- __meta_kubernetes_pod_container_port_name
|
|
regex: metrics
|
|
action: keep
|
|
- source_labels:
|
|
- __meta_kubernetes_pod_container_name
|
|
target_label: job
|
|
authorization:
|
|
credentials_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|