From 10cdf9408d3f029c203a01c48e27a08f8605366d Mon Sep 17 00:00:00 2001 From: "Emily (Agent)" Date: Sun, 9 Aug 2026 16:15:35 -0500 Subject: [PATCH] feat: enable request-level logging for bandwidth/traffic incident tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Triggered by investigating a several-hour >10Mbps traffic spike to linode. HAProxy's own IPAccounting confirmed ~121GB moved over ~19.6h before it crash-looped, but with 'option httplog' commented out and no per-backend request logs, there was no way to attribute that traffic to a specific backend, host, or client. - linode: enable HAProxy httplog + defaults 'log global' (was commented out) so every proxied HTTP request is now logged with timing/status/bytes. - linode: add a haproxy 'stats' listener on 127.0.0.1:8404 for live per-backend/per-server connection and byte counters. - linode: route nginx (Nextcloud's local vhost) access logs to journald via syslog, since the read-only monitoring account has no access to /var/log/nginx/*. - linode: enable vnstat for historical per-interface bandwidth tracking (5-min granularity) so a reported 'traffic was high for N hours' can be confirmed/timestamped immediately instead of reconstructed after the fact from journal timestamps. - k3s manifests: enable Traefik access logging (JSON) — this is the ingress layer HAProxy forwards :80 traffic to (git/matrix/immich), and lacked any per-request visibility. - hosts/baseline.nix (fleet-wide): add a journald rate limit (2000 lines / 30s per unit). Found live while investigating that uptime-kuma on 'kuma' was logging a Prometheus label-validation error on every monitor beat (~100k lines/hour), which was itself degrading journalctl responsiveness on that host during the cross-host traffic scan. Related but not otherwise addressed here: Nebula relay/handshake churn on kuma's tunnel and the etcd read-latency warnings seen on isaiah/zeke around the same incident window — noted for a future investigation, not fixed by this PR. --- hosts/baseline.nix | 13 +++++++++++ hosts/unstable/linode/default.nix | 39 +++++++++++++++++++++++++------ manifests/helm/traefik.yaml | 11 +++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/hosts/baseline.nix b/hosts/baseline.nix index 0c03769..b42c329 100644 --- a/hosts/baseline.nix +++ b/hosts/baseline.nix @@ -101,6 +101,19 @@ # Enable the OpenSSH daemon for remote control services = { locate.enable = true; + + # Defensive rate-limit: cap any single misbehaving service's journal + # output fleet-wide. Discovered live on kuma (uptime-kuma logging a + # Prometheus-label validation error on every monitor beat, ~100k + # lines/hour) that a runaway logger can itself become the obstacle to + # incident investigation — journalctl becomes slow/unresponsive and + # disk fills — on top of drowning out genuinely useful log signal. + # This doesn't fix a specific app's bug, but bounds the blast radius. + journald.extraConfig = '' + RateLimitIntervalSec=30s + RateLimitBurst=2000 + ''; + niks3-auto-upload = { enable = config.greg.nix.cache; authTokenFile = config.age.secrets.niks3-api-token.path; diff --git a/hosts/unstable/linode/default.nix b/hosts/unstable/linode/default.nix index de5a86a..2c503e6 100644 --- a/hosts/unstable/linode/default.nix +++ b/hosts/unstable/linode/default.nix @@ -49,6 +49,14 @@ in pkgs'.upgrade-pg-cluster ]; + # Historical per-interface bandwidth tracking (5-min granularity, kept for + # months). This is what's actually missing when diagnosing "traffic was + # high for the past several hours" reports after the fact — journalctl + # timestamps only tell you what else was happening, not the traffic curve + # itself. `vnstat -h`/`vnstat --json h` gives an immediate confirm/deny of + # a reported window without waiting on live sampling. + services.vnstat.enable = true; + greg = { backup.jobs = { nextcloud-bkup = { @@ -164,6 +172,7 @@ in log /dev/log local0 defaults + log global timeout connect 500s timeout client 500s timeout server 1h @@ -176,6 +185,12 @@ in server git-jeremiah jeremiah.thehellings.lan:32222 server git-zeke zeke.thehellings.lan:32222 + listen stats + bind 127.0.0.1:8404 + stats enable + stats uri / + stats refresh 10s + frontend https bind *:80 bind *:443 ssl crt ${config.security.acme.certs."thehellings.com".directory}/full.pem @@ -187,8 +202,8 @@ in option http-server-close option http-keep-alive + option httplog - #option httplog #declare capture response len 80 #http-response capture res.hdr(Location) id 0 @@ -300,12 +315,22 @@ in }; # Move to :8080 so that we can run haproxy as the primary HTTP service - nginx.virtualHosts."${config.services.nextcloud.hostName}".listen = [ - { - addr = "127.0.0.1"; - port = nextcloudPort; - } - ]; + nginx = { + virtualHosts."${config.services.nextcloud.hostName}".listen = [ + { + addr = "127.0.0.1"; + port = nextcloudPort; + } + ]; + + # Route nginx access logs through syslog/journald (rather than only to + # /var/log/nginx/access.log, which the read-only monitoring account + # can't read) so `journalctl -t nginx-access` gives visibility into + # Nextcloud request traffic during bandwidth investigations. + appendHttpConfig = '' + access_log syslog:server=unix:/dev/log,tag=nginx-access combined; + ''; + }; openssh.settings.PasswordAuthentication = false; diff --git a/manifests/helm/traefik.yaml b/manifests/helm/traefik.yaml index ee24569..0d226ea 100644 --- a/manifests/helm/traefik.yaml +++ b/manifests/helm/traefik.yaml @@ -10,6 +10,16 @@ spec: - "--api.dashboard=true" - "--api.insecure=true" - "--log.level=DEBUG" + # Access logging: gives per-request visibility (client IP, host, + # path, bytes, duration) for every ingress route Traefik terminates + # (git.k3s.thehellings.lan, matrix.k3s.thehellings.lan, immich, etc). + # This is the layer HAProxy on linode forwards :80 traffic to, so + # having request-level logs here is essential for tracing bandwidth + # spikes back to a specific host/path/client rather than just a + # backend-level byte count. + - "--accesslog=true" + - "--accesslog.format=json" + - "--accesslog.fields.headers.defaultmode=keep" ports: postgres: expose: @@ -20,3 +30,4 @@ spec: traefik: expose: default: true +