fix: remove pgadmin from Kubernetes

DAVx5 (CalDAV/CardDAV) reported the exact same 'unexpected end of
stream' / EOFException error again at 2026-08-10T04:00:58Z, roughly
15 minutes after PR #39 deployed. That PR's backend-side fix (option
http-server-close on 'backend next') is confirmed working -
journalctl/nginx access logs show a completely clean, uninterrupted
request stream on the haproxy<->nginx leg through the exact failure
timestamp.

Root cause of the recurrence: PR #39 also added 'timeout
http-keep-alive 30s' to defaults, intended as an unrelated tidy-up
given maxconn=80. That value didn't account for client-side HTTP
connection pooling: DAVx5 runs on OkHttp, which holds idle pooled
connections open for up to 5 minutes by default before evicting them.
With haproxy closing idle client-facing keep-alive connections after
just 30s, any DAVx5 connection idle between 30s-300s got silently
closed by haproxy while the client still considered it live - the
client's next reuse attempt produced exactly the same class of error,
just relocated from the haproxy<->nginx leg to the client<->haproxy
leg instead of being fixed.

Fix:
- defaults: raise 'timeout http-keep-alive' from 30s to 6m, safely
  above OkHttp's 300s (5min) idle-eviction default, so a client's own
  pool always evicts a stale connection before haproxy would.
- backend next: add 'log-tag next' so this backend's haproxy log
  lines carry a distinct syslog tag ('journalctl -t next') instead of
  being interleaved with every other backend under the shared
  'haproxy' tag - this specific incident took significant manual
  grep/awk work to isolate 'next' traffic from git/matrix/immich noise
  in the same log stream, which a dedicated tag eliminates going
  forward.

Verified by comparing haproxy's own next/nextcloud access log lines
(all showing normal termination, no CD/SD flags) against nginx's
nginx_access journal (clean, continuous, no gap) across the exact
04:00:58 UTC failure window - confirming the backend-side legs were
healthy and the failure had to be on the client<->haproxy leg instead.

Could not run 'haproxy -c' locally (no toolchain in the agent
sandbox) - recommend confirming via CI/garnix before merge, same
caveat as prior PRs in this series (#37, #38, #39).

darwin/hosts/MacBook-Prolocal

@@ -1 +1 @@
-gregory
+ivr

darwin/hosts/gregory

@@ -0,0 +1 @@
+ivr

darwin/hosts/gregory.hellings-mbp

@@ -1 +1 @@
-gregory
+ivr

darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024

@@ -0,0 +1 @@
+lithic

darwin/hosts/ivr

@@ -1 +0,0 @@
-gregory/

home/hosts/builder2/default.nix

@@ -1 +0,0 @@
-{...}: {}

home/ssh/emily_authorized_keys

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBUz4YsVKBERXDT9nl4lwWHoA7NkI7M1Wr3QEYtgz9hy emily-monitoring@thehellings.com

hosts/baseline.nix

@@ -101,6 +101,19 @@
   # Enable the OpenSSH daemon for remote control
   services = {
     locate.enable = true;
+
+    # Defensive rate-limit: cap any single misbehaving service's journal
+    # output fleet-wide. Discovered live on kuma (uptime-kuma logging a
+    # Prometheus-label validation error on every monitor beat, ~100k
+    # lines/hour) that a runaway logger can itself become the obstacle to
+    # incident investigation — journalctl becomes slow/unresponsive and
+    # disk fills — on top of drowning out genuinely useful log signal.
+    # This doesn't fix a specific app's bug, but bounds the blast radius.
+    journald.extraConfig = ''
+      RateLimitIntervalSec=30s
+      RateLimitBurst=2000
+    '';
+
     niks3-auto-upload = {
       enable = config.greg.nix.cache;
       authTokenFile = config.age.secrets.niks3-api-token.path;

hosts/unstable/builder2/default.nix

@@ -1,60 +0,0 @@
-{
-  config,
-  metadata,
-  modulesPath,
-  ...
-}:
-{
-  # Then build nixosConfiguration.<host>.config.system.build.images.proxmox
-  # SCP that to /var/lib/vz/dumps on the Proxmox host
-  imports = [ "${modulesPath}/virtualisation/proxmox-image.nix" ];
-  greg = {
-    home = true;
-    nebula.enable = true;
-  };
-  networking = {
-    defaultGateway = metadata.infra.gw;
-    nameservers = [ metadata.infra.dns ];
-    interfaces.ens18 = {
-      useDHCP = false;
-      ipv4.addresses = [
-        {
-          address = metadata.hosts."${config.networking.hostName}".ip;
-          prefixLength = 16;
-        }
-      ];
-    };
-  };
-  virtualisation.diskSize = 20480; # Size in mebbibytes for the base disk image
-  # Use these instead of the above to run an LXC image
-  # The main reason I wouldn't use these is because Proxmox LXC does
-  # not seem to be well supported by either Nebula VPN or Tailscale,
-  # both of which I use for my mesh networking. If there isn't a need
-  # for the service to run on those networks, then by all means go ahead
-  # and use LXC!
-  # imports = [ (modulesPath + "/virtualisation/proxmox-lxc.nix") ];
-  # proxmoxLXC = {
-  #   manageNetwork = false;
-  #   privileged = true;
-  # };
-  # systemd.suppressedSystemUnits = [
-  #   "dev-mqueue.mount"
-  #   "sys-kernel-debug.mount"
-  #   "sys-fs-fuse-connections.mount"
-  # ];
-  nix.settings = {
-    sandbox = false;
-  };
-  services = {
-    fstrim.enable = false; # Let Proxmox host handle fstrim
-    openssh = {
-      enable = true;
-      openFirewall = true;
-      settings = {
-        PermitRootLogin = "yes";
-        PasswordAuthentication = true;
-        PermitEmptyPasswords = "yes";
-      };
-    };
-  };
-}

hosts/unstable/genesis/net/hosts

@@ -1,12 +1,11 @@
 # Local hosts
 10.42.0.1 switch switch.thehellings.lan  # Core switch for the network
 10.42.0.3 ap ap.thehellings.lan  # OpenWRT access point (static IP)
-10.42.0.4 joel.thehellings.lan  # Proxmox
+10.42.0.4 pve1.thehellings.lan  # Proxmox
 10.42.0.5 sanswitch.thehellings.lan  # Core switch for the SAN
 
 # Home servers
-10.42.1.1 pve1.thehellings.lan
-10.42.1.2 opnsense router opnsense.thehellings.lan router.thehellings.lan
+10.42.1.1 udm router udm.thehellings.lan router.thehellings.lan  # Ubiquiti UDM gateway
 10.42.1.3 printer.thehellings.lan
 10.42.1.4 chronicles chronicles.thehellings.lan nas.thehellings.lan s3.thehellings.lan
 10.42.1.5 genesis genesis.thehellings.lan dns dns.thehellings.lan smart smart.thehellings.lan speedtest.thehellings.lan nixcache.thehellings.lan gitcache.thehellings.lan
@@ -24,7 +23,7 @@
 10.42.4.1 matrix matrix.thehellings.lan
 
 # VIP
-10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster immich.cluster
+10.42.5.1 longhorn.cluster matrix.cluster postgres.cluster immich.cluster
 
 # IPMI
 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan
@@ -36,7 +35,7 @@
 100.70.99.91 exodus.home exodus.shire-zebra.ts.net
 100.96.198.104 genesis.home genesis.shire-zebra.ts.net smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home
 100.68.203.1 hosea.home hosea.shire-zebra.ts.net grafana.home
-100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes longhorn.kubernetes
+100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net longhorn.kubernetes
 100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes immich.kubernetes postgres.kubernetes buildbot.home
 100.90.74.19 zeke.home zeke.shire-zebra.ts.net
 100.109.86.8 linode.home linode.shire-zebra.ts.net

hosts/unstable/genesis/networking/dhcp.nix

@@ -60,7 +60,7 @@
         reservations = [
           # Static IPs for personal work
           {
-            hw-address = "00:23:24:72:64:32"; # Joel
+            hw-address = "00:23:24:72:64:32"; # PVE1
             ip-address = "10.42.0.4";
           }
           {
@@ -76,10 +76,6 @@
             #ip-address = "10.42.2.253";
             ip-address = "10.42.100.6";
           }
-          {
-            hw-address = "7c:83:34:b9:ee:ec"; # PVE1
-            ip-address = "10.42.1.1";
-          }
           {
             hw-address = "74:ee:2a:66:b3:51"; # printer
             ip-address = "10.42.1.3";

hosts/unstable/linode/default.nix

@@ -11,6 +11,21 @@ let
   homepage = "127.0.0.1:30080";
   nextcloudPort = 8080;
   sshPort = 2222;
+  matrixServer = pkgs.writeText "matrix_server" (
+    builtins.toJSON {
+      "m.server" = "matrix.thehellings.com:443";
+    }
+  );
+  matrixClient = pkgs.writeText "matrix_client" (
+    builtins.toJSON {
+      "m.homeserver" = {
+        base_url = "https://matrix.thehellings.com";
+      };
+      "m.identity_server" = {
+        base_url = "https://vector.im";
+      };
+    }
+  );
 in
 {
   imports = [
@@ -34,6 +49,14 @@ in
     pkgs'.upgrade-pg-cluster
   ];
 
+  # Historical per-interface bandwidth tracking (5-min granularity, kept for
+  # months). This is what's actually missing when diagnosing "traffic was
+  # high for the past several hours" reports after the fact — journalctl
+  # timestamps only tell you what else was happening, not the traffic curve
+  # itself. `vnstat -h`/`vnstat --json h` gives an immediate confirm/deny of
+  # a reported window without waiting on live sampling.
+  services.vnstat.enable = true;
+
   greg = {
     backup.jobs = {
       nextcloud-bkup = {
@@ -149,9 +172,30 @@ in
           log /dev/log local0
 
         defaults
+          log global
           timeout connect 500s
           timeout client 500s
           timeout server 1h
+          # HAProxy defaults to end-to-end keep-alive (client AND server side)
+          # unless a proxy overrides it. Bound how long an idle client-facing
+          # keep-alive connection is held rather than falling back to
+          # "timeout client" (500s).
+          #
+          # CORRECTION (see #39): this was originally set to 30s as a
+          # tidy-up given maxconn=80, without considering client-side
+          # connection pooling behavior. That was too aggressive: DAVx5 (and
+          # OkHttp-based HTTP clients generally) keep idle pooled
+          # connections open for up to 5 minutes client-side before
+          # eviction. With a 30s haproxy-side timeout, any client connection
+          # idle between 30s-300s got silently closed by haproxy, and the
+          # client's next reuse of it produced exactly the "unexpected end
+          # of stream"/EOFException class of error this investigation
+          # started from - just relocated from the haproxy<->nginx leg
+          # (fixed in `backend next` below) to the client<->haproxy leg.
+          # Set comfortably above OkHttp's 300s default so a client's own
+          # pool eviction always happens first and haproxy is never the one
+          # to close a connection the client still thinks is good.
+          timeout http-keep-alive 6m
 
         listen gitsshd
           bind *:${toString sshPort}
@@ -161,6 +205,12 @@ in
           server git-jeremiah jeremiah.thehellings.lan:32222
           server git-zeke zeke.thehellings.lan:32222
 
+        listen stats
+          bind 127.0.0.1:8404
+          stats enable
+          stats uri /
+          stats refresh 10s
+
         frontend https
           bind *:80
           bind *:443 ssl crt ${config.security.acme.certs."thehellings.com".directory}/full.pem
@@ -172,8 +222,8 @@ in
 
           option http-server-close
           option http-keep-alive
+          option httplog
 
-          #option httplog
           #declare capture response len 80
           #http-response capture res.hdr(Location) id 0
 
@@ -224,15 +274,29 @@ in
           option accept-unsafe-violations-in-http-response
           retries 3
           option forwardfor
+          http-request return status 200 content-type "application/json" file ${matrixClient} hdr "cache-control" "no-cache" if { path /.well-known/matrix/client }
+          http-request return status 200 content-type "application/json" file ${matrixServer} hdr "cache-control" "no-cache" if { path /.well-known/matrix/server }
           server web-container ${homepage}
 
         backend next
           log global
+          log-tag next
           mode http
           balance roundrobin
           option accept-unsafe-violations-in-http-response
           retries 3
           option forwardfor
+          # nginx (the actual listener on 127.0.0.1:8080) has
+          # keepalive_timeout 65s and will silently close an idle backend
+          # socket after that. HAProxy's default mode is end-to-end
+          # keep-alive, so without this it will happily try to reuse a
+          # backend connection nginx already closed once a mobile client's
+          # own (longer) keep-alive idle assumption outlives 65s - producing
+          # exactly the "unexpected end of stream" / EOFException the
+          # CalDAV/CardDAV client saw. Since the backend is localhost, the
+          # cost of a fresh TCP connection per request is negligible, so
+          # just don't try to reuse them here.
+          option http-server-close
           #http-response replace-value Location http://localhost:${builtins.toString nextcloudPort}/(.*) https://next.thehellings.com/\2
           server nextcloud 127.0.0.1:${builtins.toString nextcloudPort}
       '';
@@ -263,7 +327,7 @@ in
       enable = true;
       package = pkgs.nextcloud33;
       appstoreEnable = true;
-      hostName = "localhost";
+      hostName = "127.0.0.1";
       https = false;
       config = {
         adminpassFile = config.age.secrets.nextcloudadmin.path;
@@ -283,12 +347,31 @@ in
     };
 
     # Move to :8080 so that we can run haproxy as the primary HTTP service
-    nginx.virtualHosts."${config.services.nextcloud.hostName}".listen = [
-      {
-        addr = "127.0.0.1";
-        port = nextcloudPort;
-      }
-    ];
+    nginx = {
+      virtualHosts."${config.services.nextcloud.hostName}".listen = [
+        {
+          addr = "127.0.0.1";
+          port = nextcloudPort;
+        }
+      ];
+
+      # Route nginx access logs through syslog/journald (rather than only to
+      # /var/log/nginx/access.log, which the read-only monitoring account
+      # can't read) so `journalctl -t nginx_access` gives visibility into
+      # Nextcloud request traffic during bandwidth investigations.
+      #
+      # NOTE: nginx's syslog "tag" only allows alphanumeric characters and
+      # underscores (no hyphens) - an earlier version of this used
+      # tag=nginx-access, which fails nginx's config test with:
+      #   nginx: [emerg] syslog "tag" only allows alphanumeric characters
+      #   and underscore in .../nginx.conf:114
+      # That broke nginx.service (and, transitively, Nextcloud/next.thehellings.com,
+      # which is proxied through nginx on 127.0.0.1:8080) until nginx hit its
+      # systemd restart limit and gave up (start-limit-hit).
+      appendHttpConfig = ''
+        access_log syslog:server=unix:/dev/log,tag=nginx_access combined;
+      '';
+    };
 
     openssh.settings.PasswordAuthentication = false;
 

manifests/databases/kustomization.yaml

@@ -4,6 +4,5 @@ resources:
   - namespace.yaml
   - secrets.yaml
   - postgres-cluster.yaml
-  - postgres-pgadmin.yaml
   - postgres-matrix.yaml
   - ingress.yaml

manifests/databases/postgres-cluster.yaml

@@ -11,13 +11,6 @@ spec:
 
   managed:
     roles:
-      - name: pgadmin
-        ensure: present
-        comment: PG Admin user
-        login: true
-        superuser: true
-        passwordSecret:
-          name: postgres-user-pgadmin
       - name: matrix
         ensure: present
         comment: Matrix DB user

manifests/databases/postgres-pgadmin.yaml

@@ -1,103 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: config-pgadmin
-data:
-  servers.json: |
-    {
-        "Servers": {
-          "1": {
-            "Name": "Postgres",
-            "Group": "Servers",
-            "Port": 5432,
-            "Username": "pgadmin",
-            "Host": "postgres-rw",
-            "SSLMode": "allow",
-            "MaintenanceDB": "postgres"
-          }
-        }
-    }
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: service-pgadmin
-spec:
-  ports:
-    - protocol: TCP
-      port: 80
-      targetPort: http
-  selector:
-    app: pgadmin
-  type: ClusterIP
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: pgadmin
-spec:
-  serviceName: service-pgadmin
-  podManagementPolicy: Parallel
-  replicas: 1
-  updateStrategy:
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      app: pgadmin
-  template:
-    metadata:
-      labels:
-        app: pgadmin
-    spec:
-      terminationGracePeriodSeconds: 10
-      containers:
-        - name: pgadmin
-          image: "dpage/pgadmin4:9.3"
-          imagePullPolicy: Always
-          env:
-            - name: PGADMIN_DEFAULT_EMAIL
-              value: greg@thehellings.com
-            - name: PGADMIN_DEFAULT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: postgres-user-pgadmin
-                  key: password
-            - name: PGADMIN_SERVER_JSON_FILE
-              value: /config-pgadmin-vol/servers.json
-          ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
-          volumeMounts:
-            - name: config-pgadmin-vol
-              mountPath: /config-pgadmin-vol/
-              readOnly: true
-            - name: pgadmin-data
-              mountPath: /var/lib/pgadmin
-      volumes:
-        - name: config-pgadmin-vol
-          configMap:
-            name: config-pgadmin
-  volumeClaimTemplates:
-    - metadata:
-        name: pgadmin-data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 3Gi
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: ingress-pgadmin
-spec:
-  ingressClassName: tailscale
-  defaultBackend:
-    service:
-      name: service-pgadmin
-      port:
-        number: 80
-  tls:
-    - hosts:
-        - pgadmin

manifests/databases/secrets.yaml

@@ -34,40 +34,6 @@ spec:
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
-metadata:
-  name: postgres-user-pgadmin
-  namespace: db
-spec:
-  target:
-    name: postgres-user-pgadmin
-    deletionPolicy: Delete
-    template:
-      type: Opaque
-      data:
-        username: |-
-          {{ .username }}
-        password: |-
-          {{ .password }}
-  data:
-    - secretKey: username
-      sourceRef:
-        storeRef:
-          name: bitwarden-login
-          kind: ClusterSecretStore
-      remoteRef:
-        key: f333d637-1667-499d-b9a0-b2e9012bd8b7
-        property: username
-    - secretKey: password
-      sourceRef:
-        storeRef:
-          name: bitwarden-login
-          kind: ClusterSecretStore
-      remoteRef:
-        key: f333d637-1667-499d-b9a0-b2e9012bd8b7
-        property: password
----
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
 metadata:
   name: k3sbackup-externalsecret
   namespace: db

manifests/gitea/chart.yaml

@@ -67,7 +67,7 @@ spec:
           RUN_MODE: dev
         server:
           DOMAIN: "shire-zebra.ts.net"
-          ROOT_URL: "https://gitea.shire-zebra.ts.net"
+          ROOT_URL: "https://git.k3s.thehellings.lan"
           SSH_PORT: "2222"
         database:
           DB_TYPE: postgres

manifests/helm/traefik.yaml

@@ -10,6 +10,16 @@ spec:
       - "--api.dashboard=true"
       - "--api.insecure=true"
       - "--log.level=DEBUG"
+      # Access logging: gives per-request visibility (client IP, host,
+      # path, bytes, duration) for every ingress route Traefik terminates
+      # (git.k3s.thehellings.lan, matrix.k3s.thehellings.lan, immich, etc).
+      # This is the layer HAProxy on linode forwards :80 traffic to, so
+      # having request-level logs here is essential for tracing bandwidth
+      # spikes back to a specific host/path/client rather than just a
+      # backend-level byte count.
+      - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.fields.headers.defaultmode=keep"
     ports:
       postgres:
         expose:
@@ -20,3 +30,4 @@ spec:
       traefik:
         expose:
           default: true
+

manifests/monitoring/config.yaml

@@ -51,8 +51,8 @@ data:
         static_configs:
           - targets:
               - "10.42.0.3"    # OpenWRT access point
-              - "10.42.0.4"    # Joel (Proxmox)
-              - "10.42.1.1"    # pve1 (Proxmox)
+              - "10.42.0.4"    # pve1 (Proxmox)
+              - "10.42.1.1"    # UDM gateway (Ubiquiti)
               - "10.42.1.4"    # chronicles (Synology NAS)
               - "10.42.1.14"   # nas1 (TrueNAS)
               - "10.42.2.57"   # odoo

modules/hm/gui/bookmarks.nix

@@ -230,7 +230,7 @@
             bookmarks = [
               {
                 name = "PVE1";
-                url = "https://10.42.1.1:8006/";
+                url = "https://10.42.0.4:8006/";
               }
               {
                 name = "Jeremiah";
@@ -253,10 +253,6 @@
                 name = "Longhorn";
                 url = "http://longhorn.shire-zebra.ts.net";
               }
-              {
-                name = "PGAdmin4";
-                url = "http://pgadmin.shire-zebra.ts.net/";
-              }
             ];
           }
           {

modules/nixos/ceph.nix

@@ -1,927 +0,0 @@
-# This is a good source for a Ceph dealio
-# https://gist.github.com0/nh2/13425a1f18b4c1ce82edb63c10b163c9
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-with lib;
-
-let
-  cfg = config.services.ceph-benaco;
-  commaSep = builtins.concatStringsSep ",";
-
-  ensureUnitExists =
-    c': name:
-    let
-    in
-    #unitName = (builtins.elemAt (builtins.split "\\." name) 0);
-    if c'.systemd.services ? unitName then name else name; # "Unable to locate ${name} at ${commaSep (builtins.attrNames c')}";
-in
-
-{
-
-  ###### interface
-
-  options = {
-
-    services.ceph-benaco = {
-
-      enable = mkEnableOption "Ceph distributed filesystem";
-
-      package = mkOption {
-        type = types.package;
-        default = pkgs.ceph;
-        defaultText = literalExpression "pkgs.ceph-benaco";
-        description = "Ceph package to use.";
-      };
-
-      fsid = mkOption {
-        type = types.str;
-        description = "Unique cluster identifier.";
-      };
-
-      clusterName = mkOption {
-        type = types.str;
-        description = "Cluster name.";
-        default = "ceph";
-      };
-
-      initialMonitors = mkOption {
-        type = types.listOf (
-          types.submodule {
-            options = {
-              hostname = mkOption {
-                type = types.str;
-                description = "Initial monitor hostname.";
-              };
-
-              ipAddress = mkOption {
-                type = types.str;
-                description = "Initial monitor IP address.";
-              };
-            };
-          }
-        );
-        description = "Initial monitors.";
-      };
-
-      mdsNodes = mkOption {
-        type = types.listOf (
-          types.submodule {
-            options = {
-              hostname = mkOption {
-                type = types.str;
-                description = "MDS hostname.";
-              };
-
-              ipAddress = mkOption {
-                type = types.str;
-                description = "MDS IP address.";
-              };
-            };
-          }
-        );
-        description = "MDS nodes.";
-      };
-
-      publicNetworks = mkOption {
-        type = types.listOf types.str;
-        description = "Public network(s) of the cluster.";
-      };
-
-      clusterNetworks = mkOption {
-        type = types.listOf types.str;
-        description = "Cluster backend networks for OSD sync";
-      };
-
-      adminKeyring = mkOption {
-        type = types.path;
-        description = "Ceph admin keyring to install on the machine.";
-      };
-
-      monitor = {
-        enable = mkEnableOption "Activate a Ceph monitor on this machine.";
-
-        initialKeyring = mkOption {
-          type = types.path;
-          description = "Keyring file to use when initializing a new monitor";
-          example = "/path/to/ceph.mon.keyring";
-        };
-
-        nodeName = mkOption {
-          type = types.str;
-          description = "Ceph monitor node name.";
-          example = "node1";
-        };
-
-        bindAddr = mkOption {
-          type = types.str;
-          description = "IP address that the OSDs shall bind to.";
-          example = "10.0.0.1";
-        };
-
-        advertisedPublicAddr = mkOption {
-          type = types.str;
-          description = "IP address that the monitor shall advertise.";
-          example = "10.0.0.1";
-        };
-      };
-
-      manager = {
-        enable = mkEnableOption "Activate a Ceph manager on this machine.";
-
-        nodeName = mkOption {
-          type = types.str;
-          description = "Ceph manager node name.";
-          example = "node1";
-        };
-      };
-
-      osdBindAddr = mkOption {
-        type = types.str;
-        description = "IP address that the OSDs shall bind to.";
-        example = "10.0.0.1";
-      };
-
-      osdAdvertisedPublicAddr = mkOption {
-        type = types.str;
-        description = "IP address that the OSDs shall advertise.";
-        example = "10.0.0.1";
-      };
-
-      osds = mkOption {
-        default = { };
-        example = {
-          osd1 = {
-            enable = true;
-            bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
-            id = 1;
-            uuid = "11111111-1111-1111-1111-111111111111";
-            blockDevice = "/dev/sdb";
-            blockDeviceUdevRuleMatcher = ''KERNEL=="sdb"'';
-            clusterAddress = "10.1.0.1";
-          };
-          osd2 = {
-            enable = true;
-            bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
-            id = 2;
-            uuid = "22222222-2222-2222-2222-222222222222";
-            blockDevice = "/dev/sdc";
-            blockDeviceUdevRuleMatcher = ''KERNEL=="sdc"'';
-            clusterAddress = "10.1.0.2";
-          };
-        };
-        description = ''
-          This option allows you to define multiple Ceph OSDs.
-          A common idiom is to use one OSD per physical hard drive.
-
-          Note that the OSD names given as attributes of this key
-          are NOT what ceph calls OSD IDs (instead, those are defined
-          by the 'services.ceph-benaco.osds.*.id' fields).
-          Instead, the name is an identifier local and unique to the
-          current machine only, used only to name the systemd service
-          for that OSD.
-        '';
-        type = types.attrsOf (
-          types.submodule {
-            options = {
-
-              enable = mkEnableOption "Activate a Ceph OSD on this machine.";
-
-              bootstrapKeyring = mkOption {
-                type = types.path;
-                description = "Ceph OSD bootstrap keyring.";
-                example = "/path/to/ceph.client.bootstrap-osd.keyring";
-              };
-
-              id = mkOption {
-                type = types.int;
-                description = "The ID of this OSD. Must be unique in the Ceph cluster.";
-                example = 1;
-              };
-
-              uuid = mkOption {
-                type = types.str;
-                description = "The UUID of this OSD. Must be unique in the Ceph cluster.";
-                example = "abcdef12-abcd-1234-abcd-1234567890ab";
-              };
-
-              systemdExtraRequiresAfter = mkOption {
-                type = types.listOf types.str;
-                default = [ ];
-                description = ''
-                  Add the specified systemd units to the "requires" and "after"
-                  lists of the systemd service of this OSD.
-
-                  Useful, for example, to decrypt the underlying block devices with LUKS first.
-
-                  NixOS modules allow override those lists from outside, but for that
-                  the names of the systemd services for the OSDs need to be known;
-                  this option is a convenience to not have to know them from outside.
-                '';
-                example = "decrypt-my-disk.service";
-              };
-
-              skipZap = mkOption {
-                type = types.bool;
-                default = false;
-                description = ''
-                  Whether to skip the zapping of the the OSD device on initial OSD
-                  installation.
-
-                  Skipping is needed because <command>ceph-volume</command> cannot
-                  zap device-mapper devices:
-                  <link xlink:href="https://tracker.ceph.com/issues/24504" />
-
-                  In that case you need to wipe the device manually.
-
-                  In the common case of placing the OSD on a cryptsetup LUKS device
-                  (which is a device-mapper device), re-creating the encryption
-                  from scratch with a new key zaps anything anyway, in which case
-                  zapping can be skipped here.
-                '';
-              };
-
-              blockDevice = mkOption {
-                type = types.str;
-                description = "The block device used to store the OSD.";
-                example = "/dev/sdb";
-              };
-
-              blockDeviceUdevRuleMatcher = mkOption {
-                type = types.str;
-                description = ''
-                  An udev rule matcher matching the block device used to store the OSD.
-                  Will be spliced into the udev rule that is
-                  used to set access permissions to the ceph user via an udev rule.
-
-                  This is a matcher instead of just a device name to allow flexibility:
-                  Normal disks can be easily matched with <code>KERNEL=="sda1"</code>, but
-                  device-mapper may not; for example, decrypted cryptsetup LUKS devices
-                  have a less useful <code>KERNEL=="dm-4"</code> and may better be matched
-                  using <code>ENV{DM_NAME}=="mydisk-decrypted"</code>.
-                '';
-                example = ''KERNEL=="sdb"'';
-              };
-
-              dbBlockDevice = mkOption {
-                type = types.nullOr types.str;
-                default = null;
-                description = ''
-                  The block device used to store the OSD's BlueStore DB device.
-
-                  Put this on a faster device than <option>blockDevice</option> to improve performance.
-
-                  See <link xlink:href="http://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/" />
-                  for details.
-                '';
-                example = "/dev/sdc";
-              };
-
-              dbBlockDeviceUdevRuleMatcher = mkOption {
-                type = types.nullOr types.str;
-                default = null;
-                description = ''
-                  Like <option>blockDeviceUdevRuleMatcher</option> but for the
-                  <option>dbBlockDevice</option>.
-                '';
-                example = ''KERNEL=="sdc"'';
-              };
-
-              clusterAddress = mkOption {
-                type = types.nullOr types.str;
-                default = null;
-                description = ''
-                  The IP address on the dedicated cluster network that
-                  is used by the backend communication for OSD communication.
-                '';
-                example = "10.1.0.1f";
-              };
-
-            };
-          }
-        );
-      };
-
-      mds = {
-        enable = mkEnableOption "Activate a Ceph MDS on this machine.";
-
-        nodeName = mkOption {
-          type = types.str;
-          description = "Ceph MDS node name.";
-          example = "node1";
-        };
-
-        listenAddr = mkOption {
-          type = types.str;
-          description = "IP address that the MDS shall advertise.";
-          example = "10.0.0.1";
-        };
-      };
-
-      extraConfig = mkOption {
-        type = types.str;
-        default = "";
-        description = ''
-          Additional ceph.conf settings.
-
-          See the sample file for inspiration:
-          <link xlink:href="https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf" />
-        '';
-      };
-    };
-  };
-
-  ###### implementation
-
-  config =
-    let
-      monDir = "/var/lib/ceph/mon/${cfg.clusterName}-${cfg.monitor.nodeName}";
-      mgrDir = "/var/lib/ceph/mgr/${cfg.clusterName}-${cfg.manager.nodeName}";
-      mdsDir = "/var/lib/ceph/mds/${cfg.clusterName}-${cfg.mds.nodeName}";
-
-      # File permissions for things that are on locations wiped at start
-      # (e.g. /run or its /var/run symlink).
-      ensureTransientCephDirs = ''
-        install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
-      '';
-
-      # File permissions from cluster deployed with ceph-deploy.
-      ensureCephDirs = ''
-        install -m 3770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/log/ceph
-        install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
-        install -m 750 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph
-        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mon
-        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mgr
-        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/osd
-      '';
-
-      # Utilities called by Ceph device health scraping, see:
-      # https://docs.ceph.com/en/latest/rados/operations/devices/#enabling-monitoring
-      # As per https://github.com/ceph/ceph-container/pull/1490/commits/c49e821599965ae92a88b2c78077ee03c4405895,
-      # both the OSDs and the `mon` need this.
-      # Ceph calls these utilities with `sudo`. That requires sudoers entries.
-      # Sudoers entries require absolute path; that exact (nix store) path needs to
-      # be used by Ceph, so it needs to be given to the systemd unit via `path`.
-      # This is why we pair each `sudoersExtraRule` with the `package` to put onto
-      # that `path`.
-      #
-      # Entries are based on:
-      # https://github.com/ceph/ceph/blob/a2f5a3c1dbfa4dce41e25da4f029a8fdb8c8d864/sudoers.d/ceph-smartctl
-      cephMonitoringSudoersCommandsAndPackages = [
-        {
-          package = pkgs.smartmontools;
-          sudoersExtraRule = {
-            # entry for `security.sudo.extraRules`
-            users = [ config.users.users.ceph.name ];
-            commands = [
-              {
-                command = "${lib.getBin pkgs.smartmontools}/bin/smartctl -x --json=o /dev/*";
-                options = [ "NOPASSWD" ];
-              }
-            ];
-          };
-        }
-        {
-          package = pkgs.nvme-cli;
-          sudoersExtraRule = {
-            # entry for `security.sudo.extraRules`
-            users = [ config.users.users.ceph.name ];
-            commands = [
-              {
-                command = "${lib.getBin pkgs.nvme-cli}/bin/nvme * smart-log-add --json /dev/*";
-                options = [ "NOPASSWD" ];
-              }
-            ];
-          };
-        }
-      ];
-
-      cephDeviceHealthMonitoringPathsOrPackages =
-        with pkgs;
-        [
-          # Contains `sudo`. Ceph wraps this around the other health check programs.
-          # Cannot use `pkgs.sudo` because that one is not SUID, see:
-          # https://discourse.nixos.org/t/sudo-uid-issues/9133
-          "/run/wrappers" # `systemd.services.<name>.path` adds the `bin/` subdir of this
-        ]
-        ++ map ({ package, ... }: package) cephMonitoringSudoersCommandsAndPackages;
-
-      # Unused localOsdServiceName in the following line
-      # deadnix: skip
-      makeCephOsdSetupSystemdService =
-        _localOsdServiceName: osdConfig:
-        let
-          osdExistenceFile = "/var/lib/ceph/osd/.${toString osdConfig.id}.${osdConfig.uuid}.nix-existence";
-        in
-        mkIf osdConfig.enable {
-          description = "Initialize Ceph OSD";
-
-          requires = osdConfig.systemdExtraRequiresAfter;
-          after = osdConfig.systemdExtraRequiresAfter;
-
-          path = with pkgs; [
-            # The following are currently missing in Ceph's wrapping, see https://github.com/NixOS/nixpkgs/issues/147801#issue-1065600852
-            util-linux # for `lsblk`
-            lvm2 # for `lvs`
-          ];
-
-          # TODO Use `udevadm trigger --settle` instead of the separate `udevadm settle`
-          #      once that feature is available to us with systemd >= 238;
-          #      see https://github.com/systemd/systemd/commit/792cc203a67edb201073351f5c766fce3d5eab45
-          preStart = ''
-            set -x
-            ${ensureCephDirs}
-            install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/bootstrap-osd
-            # `install` is not atomic, see
-            # https://lists.gnu.org/archive/html/bug-coreutils/2010-02/msg00243.html
-            # so use `mktemp` + `mv` to make it atomic.
-            TMPFILE=$(mktemp --tmpdir=/var/lib/ceph/bootstrap-osd/)
-            install -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} ${osdConfig.bootstrapKeyring} "$TMPFILE"
-            mv "$TMPFILE" /var/lib/ceph/bootstrap-osd/ceph.keyring
-
-            # Trigger udev rules for permissions of block devices and wait for them to settle.
-            udevadm trigger --name-match=${osdConfig.blockDevice}
-          ''
-          + lib.optionalString (osdConfig.dbBlockDevice != null) ''
-            udevadm trigger --name-match=${osdConfig.dbBlockDevice}
-          ''
-          + ''
-            udevadm settle
-          ''
-          + (optionalString (!osdConfig.skipZap) (
-            ''
-              # Zap OSD block devices, otherwise `ceph-osd` below will try to fsck if there's some old
-              # ceph data on the block device (see https://tracker.ceph.com/issues/24099).
-              ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.blockDevice}
-            ''
-            + lib.optionalString (osdConfig.dbBlockDevice != null) ''
-              ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.dbBlockDevice}
-            ''
-          ));
-
-          script = ''
-            set -euo pipefail
-            set -x
-            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
-            do
-              sleep 1
-            done
-
-            OSD_SECRET=$(${cfg.package}/bin/ceph-authtool --gen-print-key)
-            echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
-              ${cfg.package}/bin/ceph --cluster ${cfg.clusterName} osd new ${osdConfig.uuid} ${toString osdConfig.id} -i - \
-              -n client.bootstrap-osd -k ${osdConfig.bootstrapKeyring}
-            mkdir -p /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}
-
-            ln -s ${osdConfig.blockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block
-          ''
-          + lib.optionalString (osdConfig.dbBlockDevice != null) ''
-            ln -s ${osdConfig.dbBlockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block.db
-          ''
-          + ''
-
-            ${cfg.package}/bin/ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${toString osdConfig.id}/keyring \
-              --name osd.${toString osdConfig.id} --add-key $OSD_SECRET
-
-            ${cfg.package}/bin/ceph-osd -i ${toString osdConfig.id} --mkfs --osd-uuid ${osdConfig.uuid} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} --osd-objectstore bluestore
-            touch ${osdExistenceFile}
-          '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph, preStart as root
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${osdExistenceFile}";
-          };
-        };
-
-      makeCephOsdSystemdService =
-        localOsdServiceName: osdConfig:
-        mkIf osdConfig.enable {
-          description = "Ceph OSD";
-
-          # Note we do not have to add `osdConfig.systemdExtraRequiresAfter` here because
-          # that's already a dependency of our dependency `ceph-osd-setup-*`.
-          requires = [ (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          path = [
-            # TODO: use wrapProgram in the ceph package for this in the future
-            pkgs.getopt
-          ]
-          ++ cephDeviceHealthMonitoringPathsOrPackages;
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          preStart = ''
-            ${ensureTransientCephDirs}
-            ${lib.getLib cfg.package}/libexec/ceph/ceph-osd-prestart.sh --cluster ${cfg.clusterName} --id ${toString osdConfig.id}
-          '';
-
-          serviceConfig =
-            let
-              clusterIpArg = lib.optionalString (
-                osdConfig.clusterAddress != null
-              ) "--cluster_addr=${osdConfig.clusterAddress}";
-            in
-            {
-              LimitNOFILE = "1048576";
-              LimitNPROC = "1048576";
-
-              ExecStart = ''
-                ${cfg.package}/bin/ceph-osd -f --cluster ${cfg.clusterName} --id ${toString osdConfig.id} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.osdBindAddr}" "--public_addr=${cfg.osdAdvertisedPublicAddr}" "${clusterIpArg}"
-              '';
-              ExecReload = ''
-                ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-              '';
-              Restart = "on-failure";
-              ProtectHome = "true";
-              ProtectSystem = "full";
-              PrivateTmp = "true";
-              TasksMax = "infinity";
-              # StartLimitBurst="3";
-            };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-    in
-    mkIf cfg.enable {
-      environment.systemPackages = [ cfg.package ];
-
-      networking.firewall = {
-        allowedTCPPorts = [
-          # Ceph outside of VPN because it is very data heavy and causes packet loss.
-          # We enable msgr-v2 only because that allows its own on-wire encryption.
-          3300 # ceph msgr-v2
-        ];
-        allowedTCPPortRanges = [
-          {
-            from = 6800;
-            to = 7300;
-          } # https://docs.ceph.com/en/pacific/rados/configuration/network-config-ref/
-        ];
-      };
-
-      # Reminder of how `ceph.conf` works:
-      #
-      # * Ceph upstream docs now recommend to use underscores instead of spaces.
-      # * Options in more specific sections like `[mon]` override those in less
-      #   specific sections like `[global]`. But all options can be written into all sections,
-      #   and an option has the same name, no matter in which section it is written.
-      #   Thus, put options in `[global]`, and only use a diffent section
-      #   if you want to override an option you've set in `global`.
-      #
-      # Sample: https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf
-      environment.etc."ceph/${cfg.clusterName}.conf".text = ''
-        [global]
-        fsid = ${cfg.fsid}
-        mon_initial_members = ${commaSep (map (mon: mon.hostname) cfg.initialMonitors)}
-        mon_host = ${commaSep (map (mon: mon.ipAddress) cfg.initialMonitors)}
-
-        # Ceph clusters go into WARN health mode, until
-        # the following setting is made strict by setting it to `false`:
-        # See: https://docs.ceph.com/en/latest/security/CVE-2021-20288/#recommendations
-        # As of writing, this setting is not documented outside of the CVE note :(
-        #
-        # While for new clusters the warning no longer seems to appear, it still
-        # appears in our existing clusters unless this option is set, see:
-        # https://tracker.ceph.com/issues/53751#note-7
-        auth_allow_insecure_global_id_reclaim = false
-
-        # Disable dirfrag prefetch on MDS restart to prevent out-of-memory after
-        # many files were opened.
-        # Note this option has no effect on Ceph < 15, because it doesn't exist there.
-        # TODO: Remove this once we're on a Ceph version that includes this default,
-        #       see https://github.com/ceph/ceph/pull/44667.
-        #       This is assuming that the commit fixes existing clusters, see
-        #       https://github.com/ceph/ceph/pull/44667#issuecomment-1036103397
-        #       If it doesn't this can only be removed once we have no existing
-        #       cluster with the old default.
-        mds_oft_prefetch_dirfrags = false
-
-        # Disable sleep between HDD recovery operations, otherwise recovery
-        # will take forever when small objects (e.g. CephFS files) are on HDD.
-        # See https://tracker.ceph.com/issues/23595#note-12
-        osd_recovery_sleep_hdd = 0.0
-
-        # Increase scrub intervals by 4x.
-        # Since we store many small files on HDD, and scrubbing apparently
-        # iterates over all objects
-        # we have no chance to scrub at the default intervals.
-        #
-        # (This was written when we had 400M files across 30 HDDs.)
-        # Change this back once we have reduced our number of files per disk.
-        osd_scrub_min_interval = 345600
-        osd_scrub_max_interval = 2419200
-        osd_deep_scrub_interval = 2419200
-
-        public_network = ${commaSep cfg.publicNetworks}
-        cluster_network = ${commaSep cfg.clusterNetworks}
-        auth_cluster_required = cephx
-        auth_service_required = cephx
-        auth_client_required = cephx
-
-        # Enforce on-wire transport encryption.
-        ms_cluster_mode = secure
-        ms_service_mode = secure
-        ms_client_mode = secure
-
-        ${cfg.extraConfig}
-      '';
-
-      environment.etc."ceph/${cfg.clusterName}.client.admin.keyring" = {
-        source = cfg.adminKeyring;
-        mode = "0600";
-        # Make ceph own this keyring so that it can use it to get keys for its daemons.
-        user = "ceph";
-        group = "ceph";
-      };
-
-      users.users.ceph = {
-        isNormalUser = false;
-        isSystemUser = true;
-        # TODO: Legacy UID / GID chosen from before we configured the UID declaratively.
-        #       In the future, we whould change this whole module to use
-        #       `config.ids.uids.ceph`, like the upstream nixpkgs Ceph module does.
-        #       Switching away from `nogroup` would also be good as described there.
-        #       For both cases, we'll have to `chown` all relevant existing files on
-        #       deployments, such as `/var/lib/ceph`, and log files.
-        uid = 1001;
-        group = config.users.groups.nogroup.name;
-      };
-      users.groups.ceph = {
-        # TODO: Same TODO as above for the `uid`.
-        gid = 499;
-      };
-
-      # Allow ceph daemons (which run as user ceph) to collect device health metrics.
-      security.sudo.extraRules = map (
-        { sudoersExtraRule, ... }: sudoersExtraRule
-      ) cephMonitoringSudoersCommandsAndPackages;
-
-      # The udevadm trigger/settle in `makeCephOsdSetupSystemdService` waits for these rules rule to be applied.
-      services.udev.extraRules = lib.concatStringsSep "\n" (
-        lib.mapAttrsToList (
-          _localOsdServiceName: osdConfig:
-          ''
-            SUBSYSTEM=="block", ${osdConfig.blockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
-          ''
-          + lib.optionalString (osdConfig.dbBlockDeviceUdevRuleMatcher != null) (''
-            SUBSYSTEM=="block", ${osdConfig.dbBlockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
-          '')
-        ) cfg.osds
-      );
-
-      systemd.services = {
-
-        ceph-mon-setup = mkIf cfg.monitor.enable {
-          description = "Initialize ceph monitor";
-
-          preStart = ensureCephDirs;
-
-          script =
-            let
-              # `--addv` seems currently required to get msgr-v2 working, see:
-              #     https://tracker.ceph.com/issues/53751#note-11
-              monmapNodes = builtins.concatStringsSep " " (
-                lib.concatMap (mon: [
-                  "--addv"
-                  mon.hostname
-                  "[v2:${mon.ipAddress}:3300,v1:${mon.ipAddress}:6789]"
-                ]) cfg.initialMonitors
-              );
-            in
-            # Monitors cannot simply be changed in config, one has to update the monmap, see note [replacing-ceph-monmap-ips-for-existing-cluster]
-            ''
-              set -euo pipefail
-              rm -rf "${monDir}" # Start from scratch.
-              echo "Initializing monitor."
-              MONMAP_DIR=`mktemp -d`
-              ${cfg.package}/bin/monmaptool --create ${monmapNodes} --fsid ${cfg.fsid} "$MONMAP_DIR/monmap"
-              ${cfg.package}/bin/ceph-mon --cluster ${cfg.clusterName} --mkfs -i ${cfg.monitor.nodeName} --monmap "$MONMAP_DIR/monmap" --keyring ${cfg.monitor.initialKeyring}
-              rm -r "$MONMAP_DIR"
-              touch ${monDir}/done
-            '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${monDir}/done";
-          };
-        };
-
-        ceph-mon = mkIf cfg.monitor.enable {
-          description = "Ceph monitor";
-
-          requires = [ (ensureUnitExists config "ceph-mon-setup.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-mon-setup.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          path = cephDeviceHealthMonitoringPathsOrPackages;
-
-          preStart = ensureTransientCephDirs;
-
-          serviceConfig = {
-            LimitNOFILE = "1048576";
-            LimitNPROC = "1048576";
-            ExecStart = ''
-              ${cfg.package}/bin/ceph-mon -f --cluster ${cfg.clusterName} --id ${cfg.monitor.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.monitor.bindAddr}" "--public_addr=${cfg.monitor.advertisedPublicAddr}"
-            '';
-            ExecReload = ''
-              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-            '';
-            PrivateDevices = "yes";
-            ProtectHome = "true";
-            ProtectSystem = "full";
-            PrivateTmp = "true";
-            TasksMax = "infinity";
-            Restart = "on-failure";
-            # StartLimitBurst="5";
-            RestartSec = "10";
-          };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-        ceph-mgr-setup = mkIf cfg.manager.enable {
-          description = "Initialize Ceph manager";
-
-          preStart = ensureCephDirs;
-
-          script = ''
-            set -euo pipefail
-            mkdir -p ${mgrDir}
-            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
-            do
-              sleep 1
-            done
-            ${cfg.package}/bin/ceph auth get-or-create mgr.${cfg.manager.nodeName} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o ${mgrDir}/keyring
-            touch "${mgrDir}/.nix_done"
-          '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${mgrDir}/.nix_done";
-          };
-        };
-
-        ceph-mgr = mkIf cfg.manager.enable {
-          description = "Ceph manager";
-
-          requires = [ (ensureUnitExists config "ceph-mgr-setup.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-mgr-setup.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          preStart = ensureTransientCephDirs;
-
-          serviceConfig = {
-            LimitNOFILE = "1048576";
-            LimitNPROC = "1048576";
-
-            ExecStart = ''
-              ${cfg.package}/bin/ceph-mgr -f --cluster ${cfg.clusterName} --id ${cfg.manager.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name}
-            '';
-            ExecReload = ''
-              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-            '';
-            Restart = "on-failure";
-            RestartSec = 10;
-            # StartLimitBurst="3";
-          };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-        ceph-mds-setup = mkIf cfg.mds.enable {
-          description = "Initialize Ceph MDS";
-
-          preStart = ensureCephDirs;
-
-          script = ''
-            set -euo pipefail
-            mkdir -p ${mdsDir}
-            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
-            do
-              sleep 1
-            done
-            ${cfg.package}/bin/ceph auth get-or-create mds.${cfg.mds.nodeName} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o ${mdsDir}/keyring
-            touch "${mdsDir}/.nix_done"
-          '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${mdsDir}/.nix_done";
-          };
-        };
-
-        ceph-mds = mkIf cfg.mds.enable {
-          description = "Ceph MDS";
-
-          requires = [ (ensureUnitExists config "ceph-mds-setup.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-mds-setup.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          preStart = ensureTransientCephDirs;
-
-          serviceConfig = {
-            LimitNOFILE = "1048576";
-            LimitNPROC = "1048576";
-
-            ExecStart = ''
-              ${cfg.package}/bin/ceph-mds -f --cluster ${cfg.clusterName} --id ${cfg.mds.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_addr=${cfg.mds.listenAddr}"
-            '';
-            ExecReload = ''
-              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-            '';
-            Restart = "on-failure";
-            # StartLimitBurst="3";
-          };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-      }
-      # Make one OSD service for each configured OSD.
-      // lib.mapAttrs' (
-        localOsdServiceName: osdConfig:
-        nameValuePair "ceph-osd-setup-${localOsdServiceName}" (
-          makeCephOsdSetupSystemdService localOsdServiceName osdConfig
-        )
-      ) cfg.osds
-      // lib.mapAttrs' (
-        localOsdServiceName: osdConfig:
-        nameValuePair "ceph-osd-${localOsdServiceName}" (
-          makeCephOsdSystemdService localOsdServiceName osdConfig
-        )
-      ) cfg.osds;
-    };
-}

modules/nixos/default.nix

@@ -7,7 +7,6 @@
     ./adblock-update.nix
     ./albyhub.nix
     ./backup.nix
-    ./ceph.nix
     ./db.nix
     ./gitea-runner.nix
     ./gnome.nix
@@ -16,6 +15,7 @@
     #./kiwix-serve.nix
     ./kubernetes.nix
     ./linode.nix
+    ./monitoring-access.nix
     ./podman.nix
     ./print.nix
     ./proxy.nix

modules/nixos/monitoring-access.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.greg.monitoring-access;
+in
+with lib;
+{
+  options.greg.monitoring-access = {
+    enable = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Create a dedicated, read-only account (`emily`) for automated
+        monitoring and analysis by the Hermes agent. The account is
+        SSH-key-only (no password set), is not added to `wheel`, and is
+        granted no sudo rights. It only gets read access to the systemd
+        journal via group membership, which is sufficient for log
+        inspection and health/analysis tasks without any privileged
+        access to the rest of the system.
+      '';
+    };
+
+    sshKeys = mkOption {
+      type = types.listOf types.str;
+      default = lib.strings.splitString "\n" (
+        builtins.readFile ../../home/ssh/emily_authorized_keys
+      );
+      description = "SSH public keys authorized to log in as the monitoring account.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups.emily = { };
+
+    users.users.emily = {
+      isNormalUser = true;
+      createHome = true;
+      description = "Read-only monitoring/analysis account (Hermes agent)";
+      group = "emily";
+      # No password is set on purpose: this account is SSH-key-only.
+      extraGroups = [
+        "systemd-journal" # read access to the journal for log analysis
+      ];
+      shell = pkgs.bashInteractive;
+      openssh.authorizedKeys.keys = cfg.sshKeys;
+    };
+  };
+}

network.json

@@ -9,10 +9,6 @@
     "tailscale": "100.64.0.0/10"
   },
   "hosts": {
-    "builder2": {
-      "ip": "10.42.1.17",
-      "system": "x86_64-linux"
-    },
     "exodus": {
       "ip": null,
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3",
@@ -135,9 +131,6 @@
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILFYyzz/9i5rXprCQj9IL1ulrbQ6E9BOSeOcvf4D/b0G",
       "tags": ["server"]
     },
-    "joel": {
-      "ip": "10.42.0.4"
-    },
     "k3s": {
       "aliases": ["*.k3s"],
       "ip": "10.42.5.1",
@@ -151,6 +144,9 @@
     "printer": {
       "ip": "10.42.1.3"
     },
+    "pve1": {
+      "ip": "10.42.0.4"
+    },
     "pve2": {
       "ip": "10.42.1.15"
     },

secrets/home.client.admin.keyring

@@ -1,6 +0,0 @@
-[client.admin]
-	key = AQBojDlmfnc8MBAAkr+PXbSewmq4OooESo2X1A==
-	caps mds = "allow *"
-	caps mgr = "allow *"
-	caps mon = "allow *"
-	caps osd = "allow *"

secrets/home.mon.keyring

@@ -1,13 +0,0 @@
-[mon.]
-	key = AQAUijlm7emnJBAAKsHT1+2EzYRQxKsL4KwwkQ==
-	caps mon = "allow *"
-[client.admin]
-	key = AQBojDlmfnc8MBAAkr+PXbSewmq4OooESo2X1A==
-	caps mds = "allow *"
-	caps mgr = "allow *"
-	caps mon = "allow *"
-	caps osd = "allow *"
-[client.bootstrap-osd]
-	key = AQDvJDpm4BDlIhAAXISJWnrOtNDk0FqhSX0/YQ==
-	caps mgr = "allow r"
-	caps mon = "profile bootstrap-osd"

secrets/home.osd-bootstrap.keyring

@@ -1,4 +0,0 @@
-[client.bootstrap-osd]
-	key = AQDvJDpm4BDlIhAAXISJWnrOtNDk0FqhSX0/YQ==
-	caps mgr = "allow r"
-	caps mon = "profile bootstrap-osd"