hosts/baseline.nix

@@ -101,6 +101,19 @@
   # Enable the OpenSSH daemon for remote control
   services = {
     locate.enable = true;
+
+    # Defensive rate-limit: cap any single misbehaving service's journal
+    # output fleet-wide. Discovered live on kuma (uptime-kuma logging a
+    # Prometheus-label validation error on every monitor beat, ~100k
+    # lines/hour) that a runaway logger can itself become the obstacle to
+    # incident investigation — journalctl becomes slow/unresponsive and
+    # disk fills — on top of drowning out genuinely useful log signal.
+    # This doesn't fix a specific app's bug, but bounds the blast radius.
+    journald.extraConfig = ''
+      RateLimitIntervalSec=30s
+      RateLimitBurst=2000
+    '';
+
     niks3-auto-upload = {
       enable = config.greg.nix.cache;
       authTokenFile = config.age.secrets.niks3-api-token.path;

hosts/unstable/linode/default.nix

@@ -49,6 +49,14 @@ in
     pkgs'.upgrade-pg-cluster
   ];
 
+  # Historical per-interface bandwidth tracking (5-min granularity, kept for
+  # months). This is what's actually missing when diagnosing "traffic was
+  # high for the past several hours" reports after the fact — journalctl
+  # timestamps only tell you what else was happening, not the traffic curve
+  # itself. `vnstat -h`/`vnstat --json h` gives an immediate confirm/deny of
+  # a reported window without waiting on live sampling.
+  services.vnstat.enable = true;
+
   greg = {
     backup.jobs = {
       nextcloud-bkup = {
@@ -164,6 +172,7 @@ in
           log /dev/log local0
 
         defaults
+          log global
           timeout connect 500s
           timeout client 500s
           timeout server 1h
@@ -176,6 +185,12 @@ in
           server git-jeremiah jeremiah.thehellings.lan:32222
           server git-zeke zeke.thehellings.lan:32222
 
+        listen stats
+          bind 127.0.0.1:8404
+          stats enable
+          stats uri /
+          stats refresh 10s
+
         frontend https
           bind *:80
           bind *:443 ssl crt ${config.security.acme.certs."thehellings.com".directory}/full.pem
@@ -187,8 +202,8 @@ in
 
           option http-server-close
           option http-keep-alive
+          option httplog
 
-          #option httplog
           #declare capture response len 80
           #http-response capture res.hdr(Location) id 0
 
@@ -300,13 +315,23 @@ in
     };
 
     # Move to :8080 so that we can run haproxy as the primary HTTP service
-    nginx.virtualHosts."${config.services.nextcloud.hostName}".listen = [
+    nginx = {
+      virtualHosts."${config.services.nextcloud.hostName}".listen = [
         {
           addr = "127.0.0.1";
           port = nextcloudPort;
         }
       ];
 
+      # Route nginx access logs through syslog/journald (rather than only to
+      # /var/log/nginx/access.log, which the read-only monitoring account
+      # can't read) so `journalctl -t nginx-access` gives visibility into
+      # Nextcloud request traffic during bandwidth investigations.
+      appendHttpConfig = ''
+        access_log syslog:server=unix:/dev/log,tag=nginx-access combined;
+      '';
+    };
+
     openssh.settings.PasswordAuthentication = false;
 
     postgresql = {

manifests/helm/traefik.yaml

@@ -10,6 +10,16 @@ spec:
       - "--api.dashboard=true"
       - "--api.insecure=true"
       - "--log.level=DEBUG"
+      # Access logging: gives per-request visibility (client IP, host,
+      # path, bytes, duration) for every ingress route Traefik terminates
+      # (git.k3s.thehellings.lan, matrix.k3s.thehellings.lan, immich, etc).
+      # This is the layer HAProxy on linode forwards :80 traffic to, so
+      # having request-level logs here is essential for tracing bandwidth
+      # spikes back to a specific host/path/client rather than just a
+      # backend-level byte count.
+      - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.fields.headers.defaultmode=keep"
     ports:
       postgres:
         expose:
@@ -20,3 +30,4 @@ spec:
       traefik:
         expose:
           default: true
+