diff --git a/hosts/baseline.nix b/hosts/baseline.nix index 0c03769..b42c329 100644 --- a/hosts/baseline.nix +++ b/hosts/baseline.nix @@ -101,6 +101,19 @@ # Enable the OpenSSH daemon for remote control services = { locate.enable = true; + + # Defensive rate-limit: cap any single misbehaving service's journal + # output fleet-wide. Discovered live on kuma (uptime-kuma logging a + # Prometheus-label validation error on every monitor beat, ~100k + # lines/hour) that a runaway logger can itself become the obstacle to + # incident investigation — journalctl becomes slow/unresponsive and + # disk fills — on top of drowning out genuinely useful log signal. + # This doesn't fix a specific app's bug, but bounds the blast radius. + journald.extraConfig = '' + RateLimitIntervalSec=30s + RateLimitBurst=2000 + ''; + niks3-auto-upload = { enable = config.greg.nix.cache; authTokenFile = config.age.secrets.niks3-api-token.path; diff --git a/hosts/unstable/linode/default.nix b/hosts/unstable/linode/default.nix index de5a86a..2c503e6 100644 --- a/hosts/unstable/linode/default.nix +++ b/hosts/unstable/linode/default.nix @@ -49,6 +49,14 @@ in pkgs'.upgrade-pg-cluster ]; + # Historical per-interface bandwidth tracking (5-min granularity, kept for + # months). This is what's actually missing when diagnosing "traffic was + # high for the past several hours" reports after the fact — journalctl + # timestamps only tell you what else was happening, not the traffic curve + # itself. `vnstat -h`/`vnstat --json h` gives an immediate confirm/deny of + # a reported window without waiting on live sampling. + services.vnstat.enable = true; + greg = { backup.jobs = { nextcloud-bkup = { @@ -164,6 +172,7 @@ in log /dev/log local0 defaults + log global timeout connect 500s timeout client 500s timeout server 1h @@ -176,6 +185,12 @@ in server git-jeremiah jeremiah.thehellings.lan:32222 server git-zeke zeke.thehellings.lan:32222 + listen stats + bind 127.0.0.1:8404 + stats enable + stats uri / + stats refresh 10s + frontend https bind *:80 bind *:443 ssl crt ${config.security.acme.certs."thehellings.com".directory}/full.pem @@ -187,8 +202,8 @@ in option http-server-close option http-keep-alive + option httplog - #option httplog #declare capture response len 80 #http-response capture res.hdr(Location) id 0 @@ -300,12 +315,22 @@ in }; # Move to :8080 so that we can run haproxy as the primary HTTP service - nginx.virtualHosts."${config.services.nextcloud.hostName}".listen = [ - { - addr = "127.0.0.1"; - port = nextcloudPort; - } - ]; + nginx = { + virtualHosts."${config.services.nextcloud.hostName}".listen = [ + { + addr = "127.0.0.1"; + port = nextcloudPort; + } + ]; + + # Route nginx access logs through syslog/journald (rather than only to + # /var/log/nginx/access.log, which the read-only monitoring account + # can't read) so `journalctl -t nginx-access` gives visibility into + # Nextcloud request traffic during bandwidth investigations. + appendHttpConfig = '' + access_log syslog:server=unix:/dev/log,tag=nginx-access combined; + ''; + }; openssh.settings.PasswordAuthentication = false; diff --git a/manifests/helm/traefik.yaml b/manifests/helm/traefik.yaml index ee24569..0d226ea 100644 --- a/manifests/helm/traefik.yaml +++ b/manifests/helm/traefik.yaml @@ -10,6 +10,16 @@ spec: - "--api.dashboard=true" - "--api.insecure=true" - "--log.level=DEBUG" + # Access logging: gives per-request visibility (client IP, host, + # path, bytes, duration) for every ingress route Traefik terminates + # (git.k3s.thehellings.lan, matrix.k3s.thehellings.lan, immich, etc). + # This is the layer HAProxy on linode forwards :80 traffic to, so + # having request-level logs here is essential for tracing bandwidth + # spikes back to a specific host/path/client rather than just a + # backend-level byte count. + - "--accesslog=true" + - "--accesslog.format=json" + - "--accesslog.fields.headers.defaultmode=keep" ports: postgres: expose: @@ -20,3 +30,4 @@ spec: traefik: expose: default: true +