Triggered by investigating a several-hour >10Mbps traffic spike to linode. HAProxy's own IPAccounting confirmed ~121GB moved over ~19.6h before it crash-looped, but with 'option httplog' commented out and no per-backend request logs, there was no way to attribute that traffic to a specific backend, host, or client. - linode: enable HAProxy httplog + defaults 'log global' (was commented out) so every proxied HTTP request is now logged with timing/status/bytes. - linode: add a haproxy 'stats' listener on 127.0.0.1:8404 for live per-backend/per-server connection and byte counters. - linode: route nginx (Nextcloud's local vhost) access logs to journald via syslog, since the read-only monitoring account has no access to /var/log/nginx/*. - linode: enable vnstat for historical per-interface bandwidth tracking (5-min granularity) so a reported 'traffic was high for N hours' can be confirmed/timestamped immediately instead of reconstructed after the fact from journal timestamps. - k3s manifests: enable Traefik access logging (JSON) — this is the ingress layer HAProxy forwards :80 traffic to (git/matrix/immich), and lacked any per-request visibility. - hosts/baseline.nix (fleet-wide): add a journald rate limit (2000 lines / 30s per unit). Found live while investigating that uptime-kuma on 'kuma' was logging a Prometheus label-validation error on every monitor beat (~100k lines/hour), which was itself degrading journalctl responsiveness on that host during the cross-host traffic scan. Related but not otherwise addressed here: Nebula relay/handshake churn on kuma's tunnel and the etcd read-latency warnings seen on isaiah/zeke around the same incident window — noted for a future investigation, not fixed by this PR.
190 lines
4.2 KiB
Nix
190 lines
4.2 KiB
Nix
{
|
||
config,
|
||
lib,
|
||
metadata,
|
||
pkgs,
|
||
top,
|
||
...
|
||
}:
|
||
{
|
||
imports = [
|
||
../modules/nix-conf.nix
|
||
top.niks3.nixosModules.niks3-auto-upload
|
||
];
|
||
|
||
age.secrets.niks3-api-token.file = ../secrets/niks3/api_token.age;
|
||
|
||
console = {
|
||
font = "Lat2-Terminus16";
|
||
keyMap = "us";
|
||
};
|
||
|
||
environment.systemPackages = with pkgs; [
|
||
top.niks3.packages.${pkgs.stdenv.hostPlatform.system}.niks3
|
||
agenix
|
||
bitwarden-cli
|
||
bmon
|
||
btop
|
||
btrfs-progs
|
||
coreutils-full
|
||
diffutils
|
||
efibootmgr
|
||
findutils
|
||
file
|
||
git
|
||
gnupatch
|
||
top.self.packages.${pkgs.stdenv.hostPlatform.system}.hms # My own home manager switcher
|
||
iperf
|
||
killall
|
||
nano
|
||
nfs-utils
|
||
lshw
|
||
pciutils
|
||
psmisc
|
||
pwgen
|
||
unzip
|
||
usbutils
|
||
wget
|
||
xfsprogs
|
||
];
|
||
|
||
i18n.defaultLocale = "en_US.UTF-8";
|
||
|
||
nix = {
|
||
gc.dates = "weekly";
|
||
settings.auto-optimise-store = true;
|
||
};
|
||
|
||
# Network Manager pulls in too many deps
|
||
networking = {
|
||
extraHosts =
|
||
let
|
||
onNetwork =
|
||
attr: _k: v:
|
||
(builtins.hasAttr attr v) && v.${attr} != null;
|
||
getIPs =
|
||
attr: domain:
|
||
(lib.mapAttrsToList (host: v: "${builtins.getAttr attr v} ${host}.${domain}") (
|
||
lib.filterAttrs (onNetwork attr) metadata.hosts
|
||
));
|
||
in
|
||
builtins.concatStringsSep "\n" (
|
||
(getIPs "ts" "shire-zebra.ts.net")
|
||
++ (getIPs "nebulaIp" "nebula.thehellings.com")
|
||
++ (getIPs "nebulaIp" "nebula")
|
||
++ (getIPs "ip" "thehellings.lan")
|
||
);
|
||
search = [
|
||
"nebula.thehellings.com"
|
||
];
|
||
networkmanager.enable = false;
|
||
};
|
||
|
||
programs = {
|
||
gnupg.agent.enable = true;
|
||
|
||
xonsh = {
|
||
enable = true;
|
||
};
|
||
|
||
ssh = {
|
||
knownHosts = builtins.mapAttrs (n: v: {
|
||
extraHostNames = [
|
||
"${n}.thehellings.lan"
|
||
"${n}.shire-zebra.ts.net"
|
||
];
|
||
publicKey = v.pubkey;
|
||
}) (lib.filterAttrs (_: v: v ? "pubkey") metadata.hosts);
|
||
};
|
||
};
|
||
|
||
# Enable the OpenSSH daemon for remote control
|
||
services = {
|
||
locate.enable = true;
|
||
|
||
# Defensive rate-limit: cap any single misbehaving service's journal
|
||
# output fleet-wide. Discovered live on kuma (uptime-kuma logging a
|
||
# Prometheus-label validation error on every monitor beat, ~100k
|
||
# lines/hour) that a runaway logger can itself become the obstacle to
|
||
# incident investigation — journalctl becomes slow/unresponsive and
|
||
# disk fills — on top of drowning out genuinely useful log signal.
|
||
# This doesn't fix a specific app's bug, but bounds the blast radius.
|
||
journald.extraConfig = ''
|
||
RateLimitIntervalSec=30s
|
||
RateLimitBurst=2000
|
||
'';
|
||
|
||
niks3-auto-upload = {
|
||
enable = config.greg.nix.cache;
|
||
authTokenFile = config.age.secrets.niks3-api-token.path;
|
||
serverUrl = "http://hosea.nebula.thehellings.com:5751";
|
||
verifyS3Integrity = true;
|
||
};
|
||
openssh = {
|
||
enable = true;
|
||
settings.X11Forwarding = true;
|
||
};
|
||
prometheus.exporters = {
|
||
node = {
|
||
enable = true;
|
||
enabledCollectors = [
|
||
"ethtool"
|
||
"logind"
|
||
"mountstats"
|
||
"systemd"
|
||
"tcpstat"
|
||
];
|
||
};
|
||
ping = {
|
||
enable = true;
|
||
settings = {
|
||
ping = {
|
||
interval = "10s";
|
||
timeout = "5s";
|
||
};
|
||
targets = [
|
||
"thehellings.com"
|
||
"genesis.shire-zebra.ts.net"
|
||
"www.google.com"
|
||
];
|
||
};
|
||
};
|
||
systemd.enable = true;
|
||
};
|
||
};
|
||
|
||
security = {
|
||
sudo-rs = {
|
||
enable = true;
|
||
extraRules = [
|
||
{
|
||
users = [ "greg" ];
|
||
commands = [
|
||
{
|
||
command = "ALL";
|
||
options = [ "NOPASSWD" ];
|
||
}
|
||
];
|
||
}
|
||
];
|
||
};
|
||
sudo.enable = false;
|
||
};
|
||
|
||
# Define a user account. Don't forget to set a password with ‘passwd’.
|
||
users.users.greg = {
|
||
isNormalUser = true;
|
||
createHome = true;
|
||
extraGroups = [
|
||
"wheel"
|
||
]; # Enable ‘sudo’ for the user.
|
||
shell = config.programs.xonsh.package;
|
||
initialPassword = "password";
|
||
openssh.authorizedKeys.keys = lib.strings.splitString "\n" (
|
||
builtins.readFile ../home/ssh/authorized_keys
|
||
);
|
||
};
|
||
|
||
system.stateVersion = "24.11";
|
||
}
|