root
ab6e1d24fe
- Add restic_backups and mattermost scrape jobs to Prometheus config - Add UnPoller deployment+service+secret for UniFi metrics (unpoller.yaml) - Add unifi scrape job targeting unpoller in-cluster - Add 4 Grafana dashboards as NixOS-managed etc files on hosea: - system-health: CPU, memory, disk, network per host - backup-health: restic backup freshness and duration - kubernetes: pod counts, restart rates, node resources - network: DNS queries, DHCP leases, UniFi ports, ping latency - Add grafana-api-token agenix secret placeholder on hosea - Open firewall ports 9100/9427/9558 for prometheus exporters in baseline.nix
423 lines
14 KiB
Nix
423 lines
14 KiB
Nix
# Edit this configuration file to define what should be installed on
|
||
# your system. Help is available in the configuration.nix(5) man page
|
||
# and in the NixOS manual (accessible by running ‘nixos-help’).
|
||
|
||
{
|
||
config,
|
||
metadata,
|
||
pkgs,
|
||
...
|
||
}:
|
||
let
|
||
wanInterface = "enp2s0";
|
||
lanInterface = "enp1s0";
|
||
lanIpAddress = metadata.hosts.${config.networking.hostName}.ip;
|
||
in
|
||
|
||
{
|
||
imports = [
|
||
# Include the results of the hardware scan.
|
||
./arr.nix
|
||
./hardware-configuration.nix
|
||
];
|
||
|
||
age.secrets.grafana-secret-key = {
|
||
file = ../../../secrets/grafana-secret-key.age;
|
||
owner = "grafana";
|
||
};
|
||
|
||
# Bootloader
|
||
boot = {
|
||
loader = {
|
||
systemd-boot.enable = true;
|
||
efi = {
|
||
canTouchEfiVariables = true;
|
||
efiSysMountPoint = "/boot/";
|
||
};
|
||
};
|
||
extraModprobeConfig = "vboxdrv";
|
||
};
|
||
|
||
environment.systemPackages = with pkgs; [
|
||
clinfo
|
||
jellyfin-ffmpeg
|
||
libva-utils
|
||
mesa-demos
|
||
radeontop
|
||
vulkan-tools
|
||
];
|
||
|
||
greg = {
|
||
home = true;
|
||
proxies = {
|
||
"jellyfin.home".target = "http://localhost:8096/";
|
||
"jellyfin.thehellings.lan".target = "http://localhost:8096/";
|
||
};
|
||
tailscale = {
|
||
enable = true;
|
||
tags = [ "home" ];
|
||
};
|
||
};
|
||
|
||
hardware = {
|
||
enableAllFirmware = true;
|
||
enableRedistributableFirmware = true;
|
||
graphics = {
|
||
enable = true;
|
||
extraPackages = with pkgs; [
|
||
intel-media-driver
|
||
intel-vaapi-driver
|
||
libva-vdpau-driver
|
||
#intel-media-sdk
|
||
intel-compute-runtime
|
||
];
|
||
};
|
||
};
|
||
|
||
networking = {
|
||
hostName = "hosea";
|
||
nameservers = [ metadata.infra.dns ];
|
||
defaultGateway = metadata.infra.gw;
|
||
interfaces = {
|
||
"${wanInterface}".useDHCP = true;
|
||
"${lanInterface}" = {
|
||
useDHCP = false;
|
||
ipv4.addresses = [
|
||
{
|
||
address = lanIpAddress;
|
||
prefixLength = 16;
|
||
}
|
||
];
|
||
};
|
||
};
|
||
};
|
||
|
||
services = {
|
||
albyhub = {
|
||
enable = true;
|
||
openFirewall = true;
|
||
settings = {
|
||
workDir = "/chain/alby";
|
||
};
|
||
};
|
||
jellyfin = {
|
||
enable = true;
|
||
openFirewall = true;
|
||
};
|
||
grafana = {
|
||
enable = true;
|
||
provision = {
|
||
enable = true;
|
||
dashboards.settings.providers = [
|
||
{
|
||
name = "Dashboards";
|
||
disableDeletion = true;
|
||
options = {
|
||
path = "/etc/grafana-dashboards";
|
||
foldersFromFilesStructure = true;
|
||
};
|
||
}
|
||
];
|
||
datasources.settings.datasources = [
|
||
{
|
||
name = "Prometheus";
|
||
type = "prometheus";
|
||
url = "https://prometheus.shire-zebra.ts.net";
|
||
isDefault = true;
|
||
editable = false;
|
||
}
|
||
];
|
||
};
|
||
settings = {
|
||
security.secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}";
|
||
server = {
|
||
domain = "${config.networking.hostName}.shire-zebra.ts.net";
|
||
enforce_domain = true;
|
||
http_addr = "0.0.0.0";
|
||
enable_gzip = true;
|
||
http_port = 3001;
|
||
};
|
||
analytics.reporting_enabled = false;
|
||
};
|
||
};
|
||
prometheus.exporters.graphite.enable = true;
|
||
# Configure keymap
|
||
xserver.xkb = {
|
||
layout = "us";
|
||
variant = "";
|
||
};
|
||
};
|
||
|
||
# After first deploy: create a Grafana service account + API token for Klaatu
|
||
# via the Grafana UI, then encrypt it: agenix -e secrets/grafana-api-token.age
|
||
age.secrets.grafana-api-token = {
|
||
file = ../../../secrets/grafana-api-token.age;
|
||
owner = "grafana";
|
||
};
|
||
|
||
environment.etc = {
|
||
"grafana-dashboards/system-health.json".text = ''
|
||
{
|
||
"uid": "system-health",
|
||
"title": "System Health Overview",
|
||
"schemaVersion": 38,
|
||
"version": 1,
|
||
"refresh": "30s",
|
||
"time": {"from": "now-3h", "to": "now"},
|
||
"panels": [
|
||
{
|
||
"id": 1,
|
||
"type": "stat",
|
||
"title": "Hosts Up",
|
||
"gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(up{job=\"node\"} == 1)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 2,
|
||
"type": "stat",
|
||
"title": "Hosts Down",
|
||
"gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(up{job=\"node\"} == 0) or vector(0)", "refId": "A"}],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]
|
||
}
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 3,
|
||
"type": "timeseries",
|
||
"title": "CPU Usage %",
|
||
"gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", "legendFormat": "{{instance}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 4,
|
||
"type": "timeseries",
|
||
"title": "Memory Usage %",
|
||
"gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "legendFormat": "{{instance}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 5,
|
||
"type": "timeseries",
|
||
"title": "Disk Usage % (/)",
|
||
"gridPos": {"x": 0, "y": 12, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"}) * 100)", "legendFormat": "{{instance}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 6,
|
||
"type": "timeseries",
|
||
"title": "Network RX bytes/s",
|
||
"gridPos": {"x": 0, "y": 20, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_network_receive_bytes_total{device!~\"lo|tailscale.*\"}[5m])", "legendFormat": "{{instance}} {{device}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 7,
|
||
"type": "timeseries",
|
||
"title": "Network TX bytes/s",
|
||
"gridPos": {"x": 12, "y": 20, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_network_transmit_bytes_total{device!~\"lo|tailscale.*\"}[5m])", "legendFormat": "{{instance}} {{device}}", "refId": "A"}]
|
||
}
|
||
]
|
||
}
|
||
'';
|
||
|
||
"grafana-dashboards/backup-health.json".text = ''
|
||
{
|
||
"uid": "backup-health",
|
||
"title": "Backup Health",
|
||
"schemaVersion": 38,
|
||
"version": 1,
|
||
"refresh": "30s",
|
||
"time": {"from": "now-3h", "to": "now"},
|
||
"panels": [
|
||
{
|
||
"id": 1,
|
||
"type": "stat",
|
||
"title": "Backup Jobs Reporting",
|
||
"gridPos": {"x": 0, "y": 0, "w": 8, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(restic_last_success_timestamp_seconds) or vector(0)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 2,
|
||
"type": "stat",
|
||
"title": "Stale Backups (>26h)",
|
||
"gridPos": {"x": 8, "y": 0, "w": 8, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(time() - restic_last_success_timestamp_seconds > 93600) or vector(0)", "refId": "A"}],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]
|
||
}
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 3,
|
||
"type": "table",
|
||
"title": "Last Backup Times",
|
||
"gridPos": {"x": 0, "y": 4, "w": 24, "h": 10},
|
||
"options": {"instant": true},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "restic_last_success_timestamp_seconds", "instant": true, "refId": "A"}],
|
||
"fieldConfig": {"defaults": {"unit": "dateTimeAsLocal"}}
|
||
},
|
||
{
|
||
"id": 4,
|
||
"type": "timeseries",
|
||
"title": "Backup Duration (seconds)",
|
||
"gridPos": {"x": 0, "y": 14, "w": 24, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "restic_last_run_duration_seconds", "refId": "A"}]
|
||
}
|
||
]
|
||
}
|
||
'';
|
||
|
||
"grafana-dashboards/kubernetes.json".text = ''
|
||
{
|
||
"uid": "kubernetes-overview",
|
||
"title": "Kubernetes Overview",
|
||
"schemaVersion": 38,
|
||
"version": 1,
|
||
"refresh": "30s",
|
||
"time": {"from": "now-3h", "to": "now"},
|
||
"panels": [
|
||
{
|
||
"id": 1,
|
||
"type": "stat",
|
||
"title": "Running Pods",
|
||
"gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Running\"}) or vector(0)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 2,
|
||
"type": "stat",
|
||
"title": "Failed Pods",
|
||
"gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Failed\"}) or vector(0)", "refId": "A"}],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]
|
||
}
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 3,
|
||
"type": "stat",
|
||
"title": "Pending Pods",
|
||
"gridPos": {"x": 12, "y": 0, "w": 6, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(kube_pod_status_phase{phase=\"Pending\"}) or vector(0)", "refId": "A"}],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"thresholds": {
|
||
"mode": "absolute",
|
||
"steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 1}]
|
||
}
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 4,
|
||
"type": "timeseries",
|
||
"title": "Pod Restart Rate",
|
||
"gridPos": {"x": 0, "y": 4, "w": 24, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "topk(10, rate(kube_pod_container_status_restarts_total[15m]) * 900)", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 5,
|
||
"type": "table",
|
||
"title": "All Pods",
|
||
"gridPos": {"x": 0, "y": 12, "w": 24, "h": 10},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kube_pod_status_phase", "instant": true, "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 6,
|
||
"type": "timeseries",
|
||
"title": "Node CPU Usage",
|
||
"gridPos": {"x": 0, "y": 22, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(node) (rate(node_cpu_seconds_total{mode=\"idle\",job=\"node\"}[5m])) * 100)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 7,
|
||
"type": "timeseries",
|
||
"title": "Node Memory Usage %",
|
||
"gridPos": {"x": 12, "y": 22, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", "refId": "A"}]
|
||
}
|
||
]
|
||
}
|
||
'';
|
||
|
||
"grafana-dashboards/network.json".text = ''
|
||
{
|
||
"uid": "network-overview",
|
||
"title": "Network & UniFi",
|
||
"schemaVersion": 38,
|
||
"version": 1,
|
||
"refresh": "30s",
|
||
"time": {"from": "now-3h", "to": "now"},
|
||
"panels": [
|
||
{
|
||
"id": 1,
|
||
"type": "stat",
|
||
"title": "DNS Queries/s",
|
||
"gridPos": {"x": 0, "y": 0, "w": 8, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(dnsmasq_queries_total[5m]) or vector(0)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 2,
|
||
"type": "stat",
|
||
"title": "DHCP Leases",
|
||
"gridPos": {"x": 8, "y": 0, "w": 8, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "kea_dhcp4_addresses_assigned_total or vector(0)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 3,
|
||
"type": "stat",
|
||
"title": "UniFi Devices",
|
||
"gridPos": {"x": 16, "y": 0, "w": 8, "h": 4},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(unifipoller_device_uptime_seconds) or vector(0)", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 4,
|
||
"type": "timeseries",
|
||
"title": "UniFi Port RX (bytes/s)",
|
||
"gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_port_receive_bytes_total[5m])", "legendFormat": "{{port_id}} {{name}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 5,
|
||
"type": "timeseries",
|
||
"title": "UniFi Port TX (bytes/s)",
|
||
"gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(unifipoller_port_transmit_bytes_total[5m])", "legendFormat": "{{port_id}} {{name}}", "refId": "A"}]
|
||
},
|
||
{
|
||
"id": 6,
|
||
"type": "timeseries",
|
||
"title": "Ping Latency (ms)",
|
||
"gridPos": {"x": 0, "y": 12, "w": 24, "h": 8},
|
||
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "probe_duration_seconds{job=\"ping\"} * 1000", "legendFormat": "{{instance}}", "refId": "A"}]
|
||
}
|
||
]
|
||
}
|
||
'';
|
||
};
|
||
|
||
users.users = {
|
||
greg.extraGroups = [ "vboxusers" ];
|
||
|
||
jellyfin.extraGroups = [
|
||
"video"
|
||
"render"
|
||
];
|
||
};
|
||
}
|