chore: update flake.lock 2026-07-19

.gitea/workflows/update-zims.yaml

@@ -1,51 +0,0 @@
-name: Update zims pin
-
-"on":
-  schedule:
-    - cron: "0 2 1 * *"  # 0200 on the first of every month
-  workflow_dispatch:
-
-jobs:
-  update-flake-lock:
-    runs-on: nix-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Update flake.lock
-        run: nix run .#zim-updater -- --output pkgs/zim/blobs.json
-
-      - name: Create PR if changed
-        env:
-          GITEA_TOKEN: ${{ secrets.KLAATU_TOKEN }}
-          GITEA_URL: https://src.thehellings.com
-          REPO: greg/nixos
-        run: |
-          if git diff --quiet pkgs/zim/blobs.json; then
-            echo "blobs.json unchanged, nothing to do"
-            exit 0
-          fi
-
-          BRANCH="auto/update-zims-$(date +%Y%m%d)"
-          git config user.email "klaatu@thehellings.com"
-          git config user.name "klaatu"
-          git checkout -b "$BRANCH"
-          git add pkgs/zim/blobs.json
-          git commit -m "chore: update zim blobs.json $(date +%Y-%m-%d)"
-
-          # Push branch using token auth
-          git remote set-url origin "https://klaatu:${GITEA_TOKEN}@${GITEA_URL#https://}/${REPO}.git"
-          git push origin "$BRANCH"
-
-          # Create PR via Gitea API
-          curl -s -X POST \
-            -H "Authorization: token ${GITEA_TOKEN}" \
-            -H "Content-Type: application/json" \
-            "${GITEA_URL}/api/v1/repos/${REPO}/pulls" \
-            -d "{
-              \"title\": \"chore: update zims $(date +%Y-%m-%d)\",
-              \"head\": \"$BRANCH\",
-              \"base\": \"main\",
-              \"body\": \"Automated monthly zims update.\\n\\nGenerated by Gitea Actions.\",
-              \"assignees\": [\"greg\"]
-            }"

ca/intermediate_ca.crt

@@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIByDCCAW+gAwIBAgIRANS+dPEH5Vqug7OWhCMmcx4wCgYIKoZIzj0EAwIwLjER
+MA8GA1UEChMISGVsbGluZ3MxGTAXBgNVBAMTEEhlbGxpbmdzIFJvb3QgQ0EwHhcN
+MjQwMjIwMjEyNzIxWhcNMzQwMjE3MjEyNzIxWjA2MREwDwYDVQQKEwhIZWxsaW5n
+czEhMB8GA1UEAxMYSGVsbGluZ3MgSW50ZXJtZWRpYXRlIENBMFkwEwYHKoZIzj0C
+AQYIKoZIzj0DAQcDQgAErZUhPfx5MpNbNVyqHDrIgUGnb6Hitl8hXlpH+kgjBzCi
+7I/+TQnl9Tc0VVNOFOYLOfBi7hV3/QudUtLGk0FKeaNmMGQwDgYDVR0PAQH/BAQD
+AgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFMZR8LDd+hNy+TmtEHvt
+zRzXboh2MB8GA1UdIwQYMBaAFAlH11TwIFGB/K75qZ3e0S+fN3JUMAoGCCqGSM49
+BAMCA0cAMEQCIBxHK6r8pMX5hrTwYKRfMmRIt44m0KtNejA2T5t09hs+AiBfvmHb
+LfoE4qoC6NgpvXorAbx+O7xkem/9svF0Ob+RsA==
+-----END CERTIFICATE-----

ca/root_ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBoTCCAUagAwIBAgIRAL/1mvE8+73nRFGsvzaaVSAwCgYIKoZIzj0EAwIwLjER
+MA8GA1UEChMISGVsbGluZ3MxGTAXBgNVBAMTEEhlbGxpbmdzIFJvb3QgQ0EwHhcN
+MjQwMjIwMjEyNzIwWhcNMzQwMjE3MjEyNzIwWjAuMREwDwYDVQQKEwhIZWxsaW5n
+czEZMBcGA1UEAxMQSGVsbGluZ3MgUm9vdCBDQTBZMBMGByqGSM49AgEGCCqGSM49
+AwEHA0IABEgNBjlT/7gmzNp9vKJvGPJn9/IizuMGVpucdrN9+J1u1ABkLNRzj5p2
+g7s3e/BG+EJnnTf/2tuq3p/wPqmQkdujRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNV
+HRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQWBBQJR9dU8CBRgfyu+amd3tEvnzdyVDAK
+BggqhkjOPQQDAgNJADBGAiEA/uBclcFCOpkEgAeBAVurktYB82sMdIyyDGHcjlwi
+pvkCIQC2kuZ/nXRjibeIebQIVTBskQ1LxlLxnx7zNSjtr3kFfQ==
+-----END CERTIFICATE-----

compose/attic.yaml

@@ -0,0 +1,54 @@
+services:
+  attic:
+    container_name: attic
+    image: ghcr.io/zhaofengli/attic:latest
+    command: ["-f", "/attic/server.toml"]
+    restart: unless-stopped
+    ports:
+      - 8080:8080
+    networks:
+      attic:
+      pgattic:
+    volumes:
+      - /mnt/all/configs/attic/server.toml:/attic/server.toml
+      - /mnt/all/containers/attic/data:/attic/storage
+    env_file:
+      - stack.env
+    depends_on:
+      pgattic:
+        condition: service_healthy
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "wget --no-verbose --tries=1 --spider http://attic:8080 || exit 1",
+        ]
+      interval: 15s
+      timeout: 10s
+      retries: 10
+      start_period: 15s
+    deploy:
+      resources:
+        reservations:
+          cpus: 1.0
+
+  pgattic:
+    container_name: pgattic
+    image: postgres:17.6-alpine
+    restart: unless-stopped
+    ports: []
+    networks:
+      pgattic:
+    volumes:
+      - /mnt/all/containers/attic/postgres:/var/lib/postgresql/data
+    env_file:
+      - stack.env
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+networks:
+  attic:
+  pgattic:

compose/pinchflat.yaml

@@ -0,0 +1,9 @@
+services:
+  pinchflat:
+    image: ghcr.io/kieraneglin/pinchflat:latest
+    ports:
+      - "8945:8945"
+    volumes:
+      - "/mnt/all/configs/pinchflat:/config"
+      - "/mnt/all/video/yt:/downloads"
+    restart: unless-stopped

compose/restic.yaml

@@ -0,0 +1,19 @@
+# Demo of rest-server with prometheus and grafana
+version: "2"
+
+services:
+  restserver:
+    image: "restic/rest-server:0.14.0"
+    volumes:
+      - /mnt/all/backups:/data
+      - /mnt/all/configs/certs:/certs
+    environment:
+      OPTIONS: >-
+        --tls
+        --tls-cert /certs/nas1.shire-zebra.ts.net.crt
+        --tls-key /certs/nas1.shire-zebra.ts.net.key
+        --path /data
+        --prometheus
+        --debug
+    ports:
+      - "30248:8000"

darwin/hosts/MacBook-Prolocal

@@ -1 +1 @@
-ivr
+gregory

darwin/hosts/gregory

@@ -1 +0,0 @@
-ivr

darwin/hosts/gregory.hellings-mbp

@@ -1 +1 @@
-ivr
+gregory

darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024

@@ -1 +0,0 @@
-lithic

darwin/hosts/ivr

@@ -0,0 +1 @@
+gregory/

flake.lock

@@ -496,16 +496,16 @@
     },
     "nixunstable": {
       "locked": {
-        "lastModified": 1784700541,
-        "narHash": "sha256-LcCdjhqwjFVrFTNW6tHm3KNYRrD1TA6bYRea30yIIjw=",
-        "owner": "geri1701",
+        "lastModified": 1784356753,
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
+        "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "3c598184d1f70c5d0beeea8b95d01ab0179e4ef7",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
         "type": "github"
       },
       "original": {
-        "owner": "geri1701",
-        "ref": "lego-v5-acme-spike",
+        "owner": "nixos",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -28,8 +28,7 @@
     nix-hardware.url = "github:nixos/nixos-hardware";
     nixpkgs-lib.url = "github:nix-community/nixpkgs.lib";
     nixvimunstable.url = "github:nix-community/nixvim/main";
-    #nixunstable.url = "github:nixos/nixpkgs/nixos-unstable";
-    nixunstable.url = "github:geri1701/nixpkgs/lego-v5-acme-spike";
+    nixunstable.url = "github:nixos/nixpkgs/nixos-unstable";
     nurpkgs.url = "github:nix-community/NUR";
     vsext.url = "github:nix-community/nix-vscode-extensions";
     wsl = {
@@ -101,7 +100,7 @@
                 {
                   deployment = {
                     inherit (v) tags;
-                    targetHost = if (v ? "connectAddr") then v.connectAddr else v.nebulaIp;
+                    targetHost = v.ts;
                     targetUser = "greg";
                   };
                 }

home/baseline/nushell/functions.nu

@@ -12,46 +12,6 @@ def --env unlock [] {
     }
 }
 
-def nebulaIps [] {
-    open /etc/nixos/network.json | get hosts | items { |h, e| $e.nebulaIp? } | where $it != null | sort
-}
-
-def localIps [] {
-    open /etc/nixos/network.json | get hosts | items { |h, e| $e.ip? } | where $it != null | sort
-}
-
-def genNebulaCert [ --ips: string, --name: string ] {
-    let public = $'~/SynologyDrive/nebula/($name).key.pub' | path expand
-    let private = $'~/SynologyDrive/nebula/($name).key' | path expand
-    let cert = $'/etc/nixos/secrets/nebula/($name).crt'
-    let ca_cert = '~/SynologyDrive/nebula/ca.crt' | path expand
-    let ca_key = '~/SynologyDrive/nebula/ca.key' | path expand
-
-    # Generate public key if there isn't one already
-    if ( not ($public | path exists) ) {
-        nebula-cert keygen -out-key $private -out-pub $public
-    }
-    # Clear old cert if there is one
-    if ( $cert | path exists) {
-        rm $cert
-    }
-
-    # Create and sign certs
-    (nebula-cert sign
-        -ca-crt $ca_cert
-        -ca-key $ca_key
-        -name $name
-        -networks $ips
-        -out-crt $cert
-        -in-pub $public
-    )
-
-    # Agenix update
-    cd /etc/nixos/secrets
-    cat $private | agenix -e $'nebula/($name).key.age'
-}
-
-
 def rebuild [ $target: string = "switch" ] {
     if (uname | get operating-system) == "Darwin" {
         sudo darwin-rebuild $target
@@ -75,8 +35,7 @@ def deploy [ $host: string, $build: string = "" ] {
     if $buildhost == "linode" or $buildhost == "genesis" {
         $buildhost = "isaiah"
     }
-    colmena apply --on $host
-    #nixos-rebuild switch --sudo --use-substitutes --target-host $host --build-host $buildhost
+    nixos-rebuild switch --sudo --use-substitutes --target-host $host --build-host $buildhost
 }
 
 def ff [ $file: string ] {

home/baseline/ssh.nix

@@ -27,12 +27,9 @@
 
         "*" = {
           DynamicForward = [ "10240" ];
-          ForwardAgent = "yes";
-          LogLevel = "error";
           ServerAliveInterval = 60;
-          SetEnv = {
-            TERM = "xterm-256color";
-          };
+          LogLevel = "error";
+          SetEnv = { TERM = "xterm-256color"; };
         };
 
         "10.42.1.4" = lib.hm.dag.entryBefore [ "10.42.*" ] nas;

home/baseline/tools.nix

@@ -8,6 +8,7 @@
   home.packages =
     with pkgs;
     [
+      attic-client
       dig
       jqp
       kubernetes-helm

home/hosts/builder2/default.nix

@@ -0,0 +1 @@
+{...}: {}

home/hosts/exodus/default.nix

@@ -23,12 +23,9 @@
     mattermost-desktop
     minio-client
     mumble
-    nebula
     nix-index
-    adoptopenjdk-icedtea-web
     pre-commit
     prismlauncher
-    rclone
     restic
     restic-browser
     tea

home/hosts/icdm-root/default.nix

@@ -0,0 +1,4 @@
+{ ... }:
+
+{
+}

home/ssh/emily_authorized_keys

@@ -1 +0,0 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBUz4YsVKBERXDT9nl4lwWHoA7NkI7M1Wr3QEYtgz9hy emily-monitoring@thehellings.com

hosts/baseline.nix

@@ -101,19 +101,6 @@
   # Enable the OpenSSH daemon for remote control
   services = {
     locate.enable = true;
-
-    # Defensive rate-limit: cap any single misbehaving service's journal
-    # output fleet-wide. Discovered live on kuma (uptime-kuma logging a
-    # Prometheus-label validation error on every monitor beat, ~100k
-    # lines/hour) that a runaway logger can itself become the obstacle to
-    # incident investigation — journalctl becomes slow/unresponsive and
-    # disk fills — on top of drowning out genuinely useful log signal.
-    # This doesn't fix a specific app's bug, but bounds the blast radius.
-    journald.extraConfig = ''
-      RateLimitIntervalSec=30s
-      RateLimitBurst=2000
-    '';
-
     niks3-auto-upload = {
       enable = config.greg.nix.cache;
       authTokenFile = config.age.secrets.niks3-api-token.path;

hosts/default.nix

@@ -30,7 +30,6 @@ let
       modules = [
         {
           nixpkgs.hostPlatform = system;
-          networking.hostName = name;
         }
         # Imported ones
         top.agenix.nixosModules.default

hosts/unstable/builder2/default.nix

@@ -0,0 +1,19 @@
+{ config, modulesPath, pkgs, lib, ... }:
+{
+  imports = [ (modulesPath + "/virtualisation/proxmox-lxc.nix") ];
+  nix.settings = { sandbox = false; };  
+  proxmoxLXC = {
+    manageNetwork = false;
+    privileged = true;
+  };
+  services.fstrim.enable = false; # Let Proxmox host handle fstrim
+  services.openssh = {
+    enable = true;
+    openFirewall = true;
+    settings = {
+        PermitRootLogin = "yes";
+        PasswordAuthentication = true;
+        PermitEmptyPasswords = "yes";
+    };
+  };
+}

hosts/unstable/exodus/default.nix

@@ -9,9 +9,13 @@
 {
   imports = [
     ./hardware-configuration.nix
-    top.nix-hardware.nixosModules.framework-intel-core-ultra-series1
+    top.nix-hardware.nixosModules.framework-11th-gen-intel
   ];
 
+  age.secrets = {
+    compose-attic.file = ../../../secrets/compose/attic.env.age;
+  };
+
   boot = {
     loader = {
       systemd-boot = {

hosts/unstable/genesis/net/hosts

@@ -1,11 +1,12 @@
 # Local hosts
 10.42.0.1 switch switch.thehellings.lan  # Core switch for the network
 10.42.0.3 ap ap.thehellings.lan  # OpenWRT access point (static IP)
-10.42.0.4 pve1.thehellings.lan  # Proxmox
+10.42.0.4 joel.thehellings.lan  # Proxmox
 10.42.0.5 sanswitch.thehellings.lan  # Core switch for the SAN
 
 # Home servers
-10.42.1.1 udm router udm.thehellings.lan router.thehellings.lan  # Ubiquiti UDM gateway
+10.42.1.1 pve1.thehellings.lan
+10.42.1.2 opnsense router opnsense.thehellings.lan router.thehellings.lan
 10.42.1.3 printer.thehellings.lan
 10.42.1.4 chronicles chronicles.thehellings.lan nas.thehellings.lan s3.thehellings.lan
 10.42.1.5 genesis genesis.thehellings.lan dns dns.thehellings.lan smart smart.thehellings.lan speedtest.thehellings.lan nixcache.thehellings.lan gitcache.thehellings.lan
@@ -23,7 +24,7 @@
 10.42.4.1 matrix matrix.thehellings.lan
 
 # VIP
-10.42.5.1 longhorn.cluster matrix.cluster postgres.cluster immich.cluster
+10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster immich.cluster
 
 # IPMI
 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan
@@ -35,7 +36,7 @@
 100.70.99.91 exodus.home exodus.shire-zebra.ts.net
 100.96.198.104 genesis.home genesis.shire-zebra.ts.net smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home
 100.68.203.1 hosea.home hosea.shire-zebra.ts.net grafana.home
-100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net longhorn.kubernetes
+100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes longhorn.kubernetes
 100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes immich.kubernetes postgres.kubernetes buildbot.home
 100.90.74.19 zeke.home zeke.shire-zebra.ts.net
 100.109.86.8 linode.home linode.shire-zebra.ts.net

hosts/unstable/genesis/networking.nix

@@ -77,7 +77,7 @@ in
       };
     };
     firewall = {
-      enable = true;
+      enable = false;
       allowedUDPPorts = [
         dhcpPort
         dnsPort
@@ -88,7 +88,7 @@ in
         80
       ];
     };
-    nftables.enable = true;
+    nftables.enable = false;
   };
 
   environment.etc."hosts.d/local".text = extraHosts;
@@ -130,10 +130,10 @@ in
         lib.mapAttrs
           (domain: net: {
             master = true;
-            file = makeZoneFile (lib'.hostsByNet net (metadata.hosts // metadata.external)) domain;
+            file = makeZoneFile (lib'.hostsByNet net metadata.hosts) domain;
           })
           {
-            "shire-zebra.ts.net" = "tailscale";
+            #"shire-zebra.ts.net" = "tailscale";
             "nebula.thehellings.com" = "nebula";
             nebula = "nebula";
             "thehellings.lan" = "lan";

hosts/unstable/genesis/networking/dhcp.nix

@@ -60,7 +60,7 @@
         reservations = [
           # Static IPs for personal work
           {
-            hw-address = "00:23:24:72:64:32"; # PVE1
+            hw-address = "00:23:24:72:64:32"; # Joel
             ip-address = "10.42.0.4";
           }
           {
@@ -76,6 +76,10 @@
             #ip-address = "10.42.2.253";
             ip-address = "10.42.100.6";
           }
+          {
+            hw-address = "7c:83:34:b9:ee:ec"; # PVE1
+            ip-address = "10.42.1.1";
+          }
           {
             hw-address = "74:ee:2a:66:b3:51"; # printer
             ip-address = "10.42.1.3";

hosts/unstable/hosea/default.nix

@@ -185,7 +185,7 @@ in
       s3 = {
         accessKeyFile = config.age.secrets.niks3-access-key-id.path;
         bucket = "niks3";
-        endpoint = "nas1.shire-zebra.ts.net:30188";
+        endpoint = "nas1.shire-zebra.ts.net:9000";
         secretKeyFile = config.age.secrets.niks3-secret-access-key.path;
         useSSL = false;
       };

hosts/unstable/icdm-root/boot.nix

@@ -0,0 +1,11 @@
+{ ... }:
+
+{
+  # Bootloader.
+  boot = {
+    loader.grub = {
+      enable = true;
+      device = "/dev/sda";
+    };
+  };
+}

hosts/unstable/icdm-root/default.nix

@@ -0,0 +1,28 @@
+# Edit this configuration file to define what should be installed on
+# your system.  Help is available in the configuration.nix(5) man page
+# and in the NixOS manual (accessible by running ‘nixos-help’).
+
+{ pkgs, ... }:
+
+{
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+    ./boot.nix
+    ./filesystem.nix
+    ./location.nix
+    ./networking.nix
+    ./wiki.nix
+  ];
+
+  # Define a user account. Don't forget to set a password with ‘passwd’.
+  users.users.greg = {
+    isNormalUser = true;
+    description = "Gregory Hellings";
+    extraGroups = [
+      "networkmanager"
+      "wheel"
+    ];
+    packages = with pkgs; [ ];
+  };
+}

hosts/unstable/icdm-root/filesystem.nix

@@ -0,0 +1,13 @@
+{ ... }:
+
+let
+in
+{
+  fileSystems."serve" = {
+    #device = "10.42.1.4:/volume1/icdm-mysql/";
+    #fsType = "nfs";
+    device = "/dev/sdb1";
+    fsType = "auto";
+    mountPoint = "/srv";
+  };
+}

hosts/unstable/icdm-root/hardware-configuration.nix

@@ -0,0 +1,53 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "xhci_pci"
+    "ehci_pci"
+    "ahci"
+    "usbhid"
+    "usb_storage"
+    "sd_mod"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/dab0d455-e25e-4445-8fa4-5320047d7e7b";
+    fsType = "btrfs";
+    options = [ "subvol=@" ];
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/5aedbb07-5761-423b-909d-2560405eae32";
+    fsType = "ext4";
+  };
+
+  fileSystems."/var" = {
+    device = "/dev/disk/by-uuid/57968536-c29d-417d-997e-85223d1d1f65";
+    fsType = "btrfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-uuid/09691dce-375a-43c6-8d40-4498d20a6d9a"; } ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.wlp2s0.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/unstable/icdm-root/location.nix

@@ -0,0 +1,15 @@
+{ ... }:
+
+{
+  # Set your time zone.
+  time.timeZone = "America/Chicago";
+
+  # Select internationalisation properties.
+  i18n.defaultLocale = "en_US.UTF-8";
+
+  # Configure keymap in X11
+  services.xserver.xkb = {
+    layout = "us";
+    variant = "";
+  };
+}

hosts/unstable/icdm-root/networking.nix

@@ -0,0 +1,63 @@
+{ ... }:
+let
+  dnsHosts = builtins.concatStringsSep "\n" [ "wiki.icdm.lan 10.42.101.1" ];
+in
+{
+  # If we have to do proxying in Bayonnais, we can start to work on that here
+  # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";
+  networking = {
+    hostName = "icdm-root";
+    useDHCP = false;
+    defaultGateway = "10.42.1.1";
+    nameservers = [
+      "100.100.100.100"
+      "10.42.1.2"
+    ];
+    enableIPv6 = false;
+
+    interfaces = {
+      eno1.ipv4.addresses = [
+        {
+          address = "10.42.101.1";
+          prefixLength = 16;
+        }
+        {
+          address = "10.77.1.2";
+          prefixLength = 16;
+        }
+      ];
+    };
+    # Allow traffic through
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 53 ];
+      allowedUDPPorts = [
+        53
+        67
+      ];
+    };
+
+    extraHosts = "${dnsHosts}";
+  };
+
+  services.dnsmasq = {
+    enable = true;
+    settings = {
+      domain = "icdm.lan";
+      dhcp-range = [ "eno1,10.77.1.10,10.77.1.255,255.255.0.0,12h" ];
+      dhcp-option = [
+        "eno1,option:router,10.77.1.1"
+        "eno1,option:dns-server,10.77.1.2,1.1.1.1"
+        "eno1,option:domain-search,icdm.lan"
+      ];
+      expand-hosts = true;
+      log-dhcp = true;
+      log-queries = true;
+      # Upstream servers
+      server = [
+        "1.1.1.1"
+        "8.8.4.4"
+      ];
+    };
+  };
+}

hosts/unstable/icdm-root/wiki.nix

@@ -0,0 +1,17 @@
+{ pkgs, ... }:
+let
+  wikiHost = "wiki.icdm.lan";
+  kiwixport = 8080;
+in
+{
+  services.kiwix-serve = {
+    enable = true;
+    port = kiwixport;
+    library = {
+      inherit (pkgs) zim;
+    };
+  };
+
+  greg.proxies."${wikiHost}".target = "http://localhost:${toString kiwixport}";
+  networking.firewall.allowedTCPPorts = [ 80 ];
+}

hosts/unstable/isaiah/default.nix

@@ -47,10 +47,6 @@
       enable = true;
       extraLabels = [ "bare-metal:host" ];
     };
-    vmdev = {
-      enable = true;
-      host = "libvirt";
-    };
   };
 
   networking = {

hosts/unstable/jeremiah/default.nix

@@ -88,7 +88,6 @@ in
       priority = 254;
     };
     nebula.enable = true;
-    proxies."buildbot.nebula.thehellings.com".target = "http://buildbot.nebula.thehellings.com:8010/";
     tailscale = {
       enable = true;
       tags = [ "home" ];
@@ -143,12 +142,12 @@ in
             updateOutputs = false;
           };
         };
-        domain = "buildbot.nebula.thehellings.com:8010";
+        domain = "${config.networking.hostName}.shire-zebra.ts.net:8010";
         evalMaxMemorySize = 8192;
         evalWorkerCount = 4;
         gitea = {
           enable = true;
-          instanceUrl = "https://src.thehellings.com";
+          instanceUrl = "https://gitea.shire-zebra.ts.net";
           oauthId = "7ec9107d-379b-47c8-870f-1191956d0500";
           oauthSecretFile = config.age.secrets.gitea-oauthSecret.path;
           tokenFile = config.age.secrets.gitea-oauthToken.path;
@@ -156,7 +155,7 @@ in
           webhookSecretFile = config.age.secrets.gitea-webhookSecret.path;
         };
         showTrace = true;
-        #webhookBaseUrl = "http://${config.networking.hostName}.nebula.thehellings.com:8010";
+        #webhookBaseUrl = "http://${config.networking.hostName}.shire-zebra.ts.net:8010";
         workersFile = config.age.secrets.gitea-buildbotWorkersFile.path;
       };
       worker = {

hosts/unstable/kuma/default.nix

@@ -1,76 +0,0 @@
-{
-  config,
-  metadata,
-  modulesPath,
-  pkgs,
-  ...
-}:
-{
-  imports = [ "${modulesPath}/virtualisation/proxmox-image.nix" ];
-  greg = {
-    home = true;
-    nebula.enable = true;
-    proxies =
-      let
-        tgt = {
-          target = "http://localhost:${config.services.uptime-kuma.settings.PORT}";
-          genAliases = false;
-        };
-      in
-      {
-        "kuma.nebula.thehellings.com" = tgt;
-        "kuma.thehellings.lan" = tgt;
-        "kuma.shire-zebra.ts.net" = tgt;
-      };
-  };
-  nix.settings = {
-    sandbox = false;
-  };
-  networking = {
-    defaultGateway = metadata.infra.gw;
-    nameservers = [ metadata.infra.dns ];
-    interfaces.ens18 = {
-      useDHCP = false;
-      ipv4.addresses = [
-        {
-          address = metadata.hosts."${config.networking.hostName}".ip;
-          prefixLength = 16;
-        }
-      ];
-    };
-  };
-  proxmox.cloudInit.enable = false;
-  services = {
-    fstrim.enable = true;
-    mysql = {
-      enable = true;
-      ensureDatabases = [
-        config.services.uptime-kuma.settings.UPTIME_KUMA_DB_NAME
-      ];
-      ensureUsers = [
-        {
-          name = config.services.uptime-kuma.settings.UPTIME_KUMA_DB_USERNAME;
-          ensurePermissions = {
-            "uptimekuma.*" = "ALL PRIVILEGES";
-          };
-        }
-      ];
-      package = pkgs.mariadb;
-    };
-    openssh = {
-      enable = true;
-      openFirewall = true;
-    };
-    uptime-kuma = {
-      enable = true;
-      settings = {
-        PORT = "3001"; # Default, but this allows us to explicitly use it elsewhere
-        UPTIME_KUMA_DB_TYPE = "mariadb";
-        UPTIME_KUMA_DB_SOCKET = "/run/mysqld/mysqld.sock";
-        UPTIME_KUMA_DB_NAME = "uptimekuma";
-        UPTIME_KUMA_DB_USERNAME = "uptimekuma";
-        UPTIME_KUMA_DB_PASSWORD = "uptimekuma";
-      };
-    };
-  };
-}

hosts/unstable/linode/default.nix

@@ -1,93 +1,34 @@
 {
-  config,
-  lib,
-  metadata,
   pkgs,
-  pkgs',
+  lib,
+  config,
   ...
 }:
 
-let
-  homepage = "127.0.0.1:30080";
-  nextcloudPort = 8080;
-  sshPort = 2222;
-  matrixServer = pkgs.writeText "matrix_server" (
-    builtins.toJSON {
-      "m.server" = "matrix.thehellings.com:443";
-    }
-  );
-  matrixClient = pkgs.writeText "matrix_client" (
-    builtins.toJSON {
-      "m.homeserver" = {
-        base_url = "https://matrix.thehellings.com";
-      };
-      "m.identity_server" = {
-        base_url = "https://vector.im";
-      };
-    }
-  );
-in
 {
   imports = [
+    ./git.nix
     ./hardware-configuration.nix
+    ./podman.nix
+    ./matrix.nix
+    ./nextcloud.nix
+    ./nginx.nix
+    ./postgres.nix
   ];
 
-  age.secrets = {
-    acme.file = ../../../secrets/acme.age;
-    nextcloudadmin = {
-      file = ../../../secrets/nextcloudadmin.age;
-      owner = "nextcloud";
-    };
-  };
-
   environment.systemPackages = with pkgs; [
     bind
     graphviz
     nix-du
     pgloader
-    podman-compose
-    pkgs'.upgrade-pg-cluster
   ];
 
-  # Historical per-interface bandwidth tracking (5-min granularity, kept for
-  # months). This is what's actually missing when diagnosing "traffic was
-  # high for the past several hours" reports after the fact — journalctl
-  # timestamps only tell you what else was happening, not the traffic curve
-  # itself. `vnstat -h`/`vnstat --json h` gives an immediate confirm/deny of
-  # a reported window without waiting on live sampling.
-  services.vnstat.enable = true;
-
   greg = {
-    backup.jobs = {
-      nextcloud-bkup = {
-        src = "/var/lib/nextcloud";
-        dest = "nextcloud-backup";
-        pre = lib.getExe (
-          pkgs.writeShellApplication {
-            name = "nextcloud-backup-pre";
-            runtimeInputs = [ config.services.nextcloud.occ ];
-            text = "nextcloud-occ maintenance:mode --on";
-          }
-        );
-        post = lib.getExe (
-          pkgs.writeShellApplication {
-            name = "nextcloud-backup-post";
-            runtimeInputs = [ config.services.nextcloud.occ ];
-            text = "nextcloud-occ maintenance:mode --off";
-          }
-        );
-      };
-      greg-postgresql-backup = {
-        src = config.services.postgresqlBackup.location;
-        dest = "linode-postgres";
-      };
-    };
     gitea-runner = {
       enable = true;
-      labels = [
+      extraLabels = [
         "vps:host"
         "blog:host"
-        "nixos-linode:host"
       ];
     };
     home = false;
@@ -95,29 +36,20 @@ in
     nebula = {
       enable = true;
       isLighthouse = true;
-      unsafeRoutes = [
-        {
-          route = "10.42.0.0/16";
-          via = metadata.hosts.genesis.nebulaIp;
-        }
-      ];
+    };
+    proxies."immich.thehellings.com" = {
+      genAliases = false;
+      target = "http://localhost:${builtins.toString config.services.immich-public-proxy.port}";
+      ssl = true;
     };
     tailscale.enable = true;
   };
 
   networking = {
-    domain = "thehellings.com";
-    firewall.allowedTCPPorts = [
-      sshPort
-      80
-      443
-    ];
-    hostName = "linode";
-    nameservers = [
-      "10.157.0.2"
-      "100.96.198.104"
-    ];
     networkmanager.enable = lib.mkForce false;
+    hostName = "linode";
+    domain = "thehellings.com";
+    nameservers = [ "100.88.91.27" ];
   };
 
   programs.ssh.extraConfig = lib.strings.concatStringsSep "\n" [
@@ -128,312 +60,10 @@ in
     "    UserKnownHostsFile /dev/null"
   ];
 
-  security.acme = {
-    acceptTerms = true;
-    defaults = {
-      dnsPropagationCheck = false;
-      dnsResolver = "92.123.95.3:53,92.123.94.3:53,92.123.94.2:53,92.123.95.4:53,92.123.95.2:53";
-      email = "greg.hellings@gmail.com";
-      extraLegoRunFlags = [ "--ipv4only" ]; # Force IPv4 only
-      #server = "https://acme-staging-v02.api.letsencrypt.org/directory";
-    };
-    certs."thehellings.com" = {
-      dnsProvider = "linode";
-      environmentFile = config.age.secrets.acme.path;
-      extraDomainNames = [
-        "*.thehellings.com"
-      ];
-    };
-  };
-
   services = {
-
-    anubis = {
-      instances = {
-        git = {
-          enable = true;
-          settings = {
-            BIND = "/run/anubis/anubis-git/anubis.sock";
-            COOKIE_DOMAIN = "thehellings.com";
-            SERVE_ROBOTS_TXT = true;
-            SLOG_LEVEL = "DEBUG";
-            TARGET = "http://git.k3s.thehellings.lan";
-          };
-        };
-      };
-    };
-
-    haproxy = {
-      enable = true;
-      config = ''
-        global
-          nbthread 4
-          maxconn 80
-          log /dev/log local0
-
-        defaults
-          log global
-          timeout connect 500s
-          timeout client 500s
-          timeout server 1h
-          # HAProxy defaults to end-to-end keep-alive (client AND server side)
-          # unless a proxy overrides it. Bound how long an idle client-facing
-          # keep-alive connection is held rather than falling back to
-          # "timeout client" (500s).
-          #
-          # CORRECTION (see #39): this was originally set to 30s as a
-          # tidy-up given maxconn=80, without considering client-side
-          # connection pooling behavior. That was too aggressive: DAVx5 (and
-          # OkHttp-based HTTP clients generally) keep idle pooled
-          # connections open for up to 5 minutes client-side before
-          # eviction. With a 30s haproxy-side timeout, any client connection
-          # idle between 30s-300s got silently closed by haproxy, and the
-          # client's next reuse of it produced exactly the "unexpected end
-          # of stream"/EOFException class of error this investigation
-          # started from - just relocated from the haproxy<->nginx leg
-          # (fixed in `backend next` below) to the client<->haproxy leg.
-          # Set comfortably above OkHttp's 300s default so a client's own
-          # pool eviction always happens first and haproxy is never the one
-          # to close a connection the client still thinks is good.
-          timeout http-keep-alive 6m
-
-        listen gitsshd
-          bind *:${toString sshPort}
-          timeout client 1h
-          mode tcp
-          server git-isaiah isaiah.thehellings.lan:32222
-          server git-jeremiah jeremiah.thehellings.lan:32222
-          server git-zeke zeke.thehellings.lan:32222
-
-        listen stats
-          bind 127.0.0.1:8404
-          stats enable
-          stats uri /
-          stats refresh 10s
-
-        frontend https
-          bind *:80
-          bind *:443 ssl crt ${config.security.acme.certs."thehellings.com".directory}/full.pem
-
-          http-request redirect scheme https unless { ssl_fc }
-          http-request add-header X-Forwarded-Proto https
-
-          http-response replace-header ^Set-Cookie:\ (.*) Set-Cookie \1;\ Secure
-
-          option http-server-close
-          option http-keep-alive
-          option httplog
-
-          #declare capture response len 80
-          #http-response capture res.hdr(Location) id 0
-
-          use_backend git if { hdr(host) -i src.thehellings.com }
-          use_backend git if { req_ssl_sni -i src.thehellings.com }
-          use_backend next if { hdr(host) -i next.thehellings.com }
-          use_backend next if { req_ssl_sni -i next.thehellings.com }
-          use_backend matrix if { hdr(host) -i matrix.thehellings.com }
-          use_backend matrix if { req_ssl_sni -i matrix.thehellings.com }
-          use_backend immich if { hdr(host) -i immich.thehellings.com }
-          use_backend immich if { req_ssl_sni -i immich.thehellings.com }
-          use_backend web if { hdr(host) -i thehellings.com }
-          use_backend web if { req_ssl_sni -i thehellings.com }
-
-        backend git
-          mode http
-          balance roundrobin
-          option accept-unsafe-violations-in-http-response
-          retries 3
-          option forwardfor
-          http-request set-header Host git.k3s.thehellings.lan
-          server git-isaiah isaiah.thehellings.lan:80
-          server git-jeremiah jeremiah.thehellings.lan:80
-          server git-zeke zeke.thehellings.lan:80
-
-        backend immich
-          mode http
-          balance roundrobin
-          option accept-unsafe-violations-in-http-response
-          retries 3
-          option forwardfor
-          server immich-proxy 127.0.0.1:${builtins.toString config.services.immich-public-proxy.port}
-
-        backend matrix
-          mode http
-          balance roundrobin
-          option accept-unsafe-violations-in-http-response
-          retries 3
-          option forwardfor
-          http-request set-header Host matrix.k3s.thehellings.lan
-          server git-isaiah isaiah.thehellings.lan:80
-          server git-jeremiah jeremiah.thehellings.lan:80
-          server git-zeke zeke.thehellings.lan:80
-
-        backend web
-          mode http
-          balance roundrobin
-          option accept-unsafe-violations-in-http-response
-          retries 3
-          option forwardfor
-          http-request return status 200 content-type "application/json" file ${matrixClient} hdr "cache-control" "no-cache" if { path /.well-known/matrix/client }
-          http-request return status 200 content-type "application/json" file ${matrixServer} hdr "cache-control" "no-cache" if { path /.well-known/matrix/server }
-          server web-container ${homepage}
-
-        backend next
-          log global
-          log-tag next
-          mode http
-          balance roundrobin
-          option accept-unsafe-violations-in-http-response
-          retries 3
-          option forwardfor
-          # nginx (the actual listener on 127.0.0.1:8080) has
-          # keepalive_timeout 65s and will silently close an idle backend
-          # socket after that. HAProxy's default mode is end-to-end
-          # keep-alive, so without this it will happily try to reuse a
-          # backend connection nginx already closed once a mobile client's
-          # own (longer) keep-alive idle assumption outlives 65s - producing
-          # exactly the "unexpected end of stream" / EOFException the
-          # CalDAV/CardDAV client saw. Since the backend is localhost, the
-          # cost of a fresh TCP connection per request is negligible, so
-          # just don't try to reuse them here.
-          option http-server-close
-          #http-response replace-value Location http://localhost:${builtins.toString nextcloudPort}/(.*) https://next.thehellings.com/\2
-          server nextcloud 127.0.0.1:${builtins.toString nextcloudPort}
-      '';
-    };
-
     immich-public-proxy = {
       enable = true;
-      immichUrl = "http://immich.k3s.thehellings.lan";
+      immichUrl = "https://immich.shire-zebra.ts.net";
     };
-
-    logrotate = {
-      enable = true;
-      settings = {
-        postgresBackup = {
-          enable = true;
-          files = "${config.services.postgresqlBackup.location}/*.gz";
-        };
-        postgresLog = {
-          enable = true;
-          files = "/var/lib/postgresql/*/log/*.log";
-          compress = true;
-          compresscmd = "${pkgs.xz}/bin/xz";
-        };
-      };
-    };
-
-    nextcloud = {
-      enable = true;
-      package = pkgs.nextcloud33;
-      appstoreEnable = true;
-      hostName = "127.0.0.1";
-      https = false;
-      config = {
-        adminpassFile = config.age.secrets.nextcloudadmin.path;
-        adminuser = "greg";
-        dbhost = "/run/postgresql";
-        dbtype = "pgsql";
-      };
-      settings = {
-        default_phone_region = "US";
-        overwriteprotocol = "http";
-        trusted_domains = [ "next.thehellings.com" ];
-        trusted_proxies = [
-          "localhost"
-          "127.0.0.1"
-        ];
-      };
-    };
-
-    # Move to :8080 so that we can run haproxy as the primary HTTP service
-    nginx = {
-      virtualHosts."${config.services.nextcloud.hostName}".listen = [
-        {
-          addr = "127.0.0.1";
-          port = nextcloudPort;
-        }
-      ];
-
-      # Route nginx access logs through syslog/journald (rather than only to
-      # /var/log/nginx/access.log, which the read-only monitoring account
-      # can't read) so `journalctl -t nginx_access` gives visibility into
-      # Nextcloud request traffic during bandwidth investigations.
-      #
-      # NOTE: nginx's syslog "tag" only allows alphanumeric characters and
-      # underscores (no hyphens) - an earlier version of this used
-      # tag=nginx-access, which fails nginx's config test with:
-      #   nginx: [emerg] syslog "tag" only allows alphanumeric characters
-      #   and underscore in .../nginx.conf:114
-      # That broke nginx.service (and, transitively, Nextcloud/next.thehellings.com,
-      # which is proxied through nginx on 127.0.0.1:8080) until nginx hit its
-      # systemd restart limit and gave up (start-limit-hit).
-      appendHttpConfig = ''
-        access_log syslog:server=unix:/dev/log,tag=nginx_access combined;
-      '';
-    };
-
-    openssh.settings.PasswordAuthentication = false;
-
-    postgresql = {
-      enable = true;
-      package = pkgs.postgresql_15;
-      checkConfig = true;
-      ensureDatabases = [ "nextcloud" ];
-      #initialScript = pkgs.writeText "create-matrix-db.sql" ''
-      #	CREATE ROLE "matrix-synapse" WITH LOGIN;
-      #	CREATE DATABASE "synapse" WITH OWNER "matrix-synapse" TEMPLATE template0 LC_COLLATE = "C" LC_CTYPE = "C";
-      #	GRANT ALL PRIVILEGES ON DATABASE "synapse" TO "matrix-synapse";
-      #'';  # These are done manually in order to set the LC_COLLATE values properly
-      ensureUsers = [
-        {
-          name = "nextcloud";
-          ensureDBOwnership = true;
-        }
-      ];
-      settings = {
-        log_connections = true;
-        log_statement = "all";
-        logging_collector = true;
-        log_filename = "postgresql.log";
-      };
-      identMap = ''
-        root root postgres
-      '';
-    };
-
-    postgresqlBackup = {
-      enable = true;
-      databases = [ "nextcloud" ];
-    };
-  };
-
-  systemd.services = {
-    haproxy = {
-      after = [
-        "nextcloud.service"
-        "network-online.target"
-      ];
-      wants = [
-        "nextcloud.service"
-        "network-online.target"
-      ];
-    };
-  };
-
-  users.users.haproxy.extraGroups = [ config.security.acme.certs."thehellings.com".group ];
-
-  # Actually serve the content from here
-  virtualisation.oci-containers = {
-    backend = "podman";
-    containers."homepage" = {
-      image = "src.thehellings.com/greg/homepage:latest";
-      ports = [ "${homepage}:80" ];
-    };
-  };
-  virtualisation.podman = {
-    enable = true;
-    dockerCompat = true;
-    dockerSocket.enable = true;
   };
 }

hosts/unstable/linode/git.nix

@@ -0,0 +1,148 @@
+{ config, ... }:
+
+let
+  srcDomain = "src.thehellings.com";
+  sshPort = 2222;
+in
+{
+  greg.proxies."${srcDomain}" = {
+    target = "http://unix:${config.services.anubis.instances.git.settings.BIND}";
+    ssl = true;
+    genAliases = false;
+    extraConfig = ''
+      #proxy_ssl_verify off;
+      #proxy_ssl_server_name on;
+
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Http-Version $server_protocol;
+      proxy_set_header User-Agent $http_user_agent;
+      client_max_body_size 100000m;
+
+      #proxy_set_header Host $host;
+      #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      #proxy_set_header X-Forwarded-Proto $scheme;
+      #proxy_set_header X-Forwarded-Ssl on;
+
+      # Ultimate AI Block List v1.7 20250924
+      # https://perishablepress.com/ultimate-ai-block-list/
+
+      if ($http_user_agent ~* "(openai\.com|\.ai|-ai|_ai|ai\.|ai-|ai_|ai=|AddSearchBot|Agentic|AgentQL|Agent\ 3|Agent\ API|AI\ Agent|AI\ Article\ Writer|AI\ Chat|AI\ Content\ Detector|AI\ Detection|AI\ Dungeon|AI\ Journalist|AI\ Legion)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(AI\ RAG|AI\ Search|AI\ SEO\ Crawler|AI\ Training|AI\ Web|AI\ Writer|AI2|AIBot|aiHitBot|AIMatrix|AISearch|AITraining|Alexa|Alice\ Yandex|AliGenie|AliyunSec|Alpha\ AI|AlphaAI|Amazon|Amelia)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(AndersPinkBot|AndiBot|Anonymous\ AI|Anthropic|AnyPicker|Anyword|Applebot|Aria\ AI|Aria\ Browse|Articoolo|Ask\ AI|AutoGen|AutoGLM|Automated\ Writer|AutoML|Autonomous\ RAG|AwarioRssBot|AwarioSmartBot|AWS\ Trainium|Azure)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(BabyAGI|BabyCatAGI|BardBot|Basic\ RAG|Bedrock|Big\ Sur|Bigsur|Botsonic|Brightbot|Browser\ MCP\ Agent|Browser\ Use|Bytebot|ByteDance|Bytespider|CarynAI|CatBoost|CC-Crawler|CCBot|Chai|Character)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Charstar\ AI|Chatbot|ChatGLM|Chatsonic|ChatUser|Chinchilla|Claude|ClearScope|Clearview|Cognitive\ AI|Cohere|Common\ Crawl|CommonCrawl|Content\ Harmony|Content\ King|Content\ Optimizer|Content\ Samurai|ContentAtScale|ContentBot|Contentedge)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(ContentShake|Conversion\ AI|Copilot|CopyAI|Copymatic|Copyscape|CoreWeave|Corrective\ RAG|Cotoyogi|CRAB|Crawl4AI|CrawlQ\ AI|Crawlspace|Crew\ AI|CrewAI|Crushon\ AI|DALL-E|DarkBard|DataFor|DataProvider)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Datenbank\ Crawler|DeepAI|Deep\ AI|DeepL|DeepMind|Deep\ Research|DeepResearch|DeepSeek|Devin|Diffbot|Doubao\ AI|DuckAssistBot|DuckDuckGo\ Chat|DuckDuckGo-Enhanced|Echobot|Echobox|Elixir|FacebookBot|FacebookExternalHit|Factset)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Falcon|FIRE-1|Firebase|Firecrawl|Flux|Flyriver|Frase\ AI|FriendlyCrawler|Gato|Gemini|Gemma|Gen\ AI|GenAI|Generative|Genspark|Gentoo-chat|Ghostwriter|GigaChat|GLM|GodMode)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Goose|GPT|Grammarly|Grendizer|Grok|GT\ Bot|GTBot|GTP|Hemingway\ Editor|Hetzner|Hugging|Hunyuan|Hybrid\ Search\ RAG|Hypotenuse\ AI|iAsk|ICC-Crawler|ImageGen|ImagesiftBot|img2dataset|imgproxy)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(INK\ Editor|INKforall|Instructor|IntelliSeek|Inferkit|ISSCyberRiskCrawler|Janitor\ AI|Jasper|Jenni\ AI|Julius\ AI|Kafkai|Kaggle|Kangaroo|Keyword\ Density\ AI|Kimi|Knowledge|KomoBot|Kruti|LangChain|Le\ Chat)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Lensa|Lightpanda|LinerBot|LLaMA|LLM|Local\ RAG\ Agent|Lovable|Magistral|magpie-crawler|Manus|MarketMuse|Meltwater|Meta-AI|Meta-External|Meta-Webindexer|Meta\ AI|MetaAI|MetaTagBot|Middleware|Midjourney)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Mini\ AGI|MiniMax|Mintlify|Mistral|Mixtral|model-training|Monica|Narrative|NeevaBot|netEstate|Neural\ Text|NeuralSEO|NinjaAI|NodeZero|Nova\ Act|NovaAct|OAI-SearchBot|OAI\ SearchBot|OASIS|Olivia)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Omgili|Open\ AI|Open\ Interpreter|OpenAGI|OpenAI|OpenBot|OpenPi|OpenRouter|OpenText\ AI|Operator|Outwrite|Page\ Analyzer\ AI|PanguBot|Panscient|Paperlibot|Paraphraser\.io|peer39_crawler|Perflexity|Perplexity|Petal)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Phind|PiplBot|PoeBot|PoeSearchBot|ProWritingAid|Proximic|Puppeteer|Python\ AI|Qualified|Quark|QuillBot|Qopywriter|Qwen|RAG\ Agent|RAG\ Azure\ AI|RAG\ Chatbot|RAG\ Database|RAG\ IS|RAG\ Pipeline|RAG\ Search)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(RAG\ with|RAG-|RAG_|Raptor|React\ Agent|Redis\ AI\ RAG|RobotSpider|Rytr|SaplingAI|SBIntuitionsBot|Scala|Scalenut|Scrap|ScriptBook|Seekr|SEObot|SEO\ Content\ Machine|SEO\ Robot|SemrushBot|Sentibot)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Serper|ShapBot|Sidetrade|Simplified\ AI|Sitefinity|Skydancer|SlickWrite|SmartBot|Sonic|Sora|Spider/2|SpiderCreator|Spin\ Rewrite|Spinbot|Stability|StableDiffusionBot|Sudowrite|SummalyBot|Super\ Agent|Superagent)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(SuperAGI|Surfer\ AI|TerraCotta|Text\ Blaze|TextCortex|Thinkbot|Thordata|TikTokSpider|Timpibot|Tinybird|Together\ AI|Traefik|TurnitinBot|uAgents|VelenPublicWebCrawler|Venus\ Chub\ AI|Vidnami\ AI|Vision\ RAG|WebSurfer|WebText)") {
+        return 444;
+      }
+      if ($http_user_agent ~* "(Webzio|WeChat|Whisper|WordAI|Wordtune|WPBot|Writecream|WriterZen|Writescope|Writesonic|xAI|xBot|YaML|YandexAdditional|YouBot|Zendesk|Zero|Zhipu|Zhuque\ AI|Zimm)") {
+        return 444;
+      }
+    '';
+  };
+  #greg.proxies."registry.thehellings.com" = {
+  #target = "https://gitea.shire-zebra.ts.net:5000";
+  #ssl = true;
+  #genAliases = false;
+  #extraConfig = ''
+  #proxy_set_header X-Forwarded-Proto https;
+  #proxy_set_header X-Forwarded-Ssl on;
+  #client_max_body_size 25000m;
+  #'';
+  #};
+
+  networking.firewall.allowedTCPPorts = [ sshPort ];
+
+  services = {
+    anubis = {
+      instances = {
+        git = {
+          enable = true;
+          settings = {
+            BIND = "/run/anubis/anubis-git/anubis.sock";
+            COOKIE_DOMAIN = "thehellings.com";
+            SERVE_ROBOTS_TXT = true;
+            TARGET = "https://gitea.shire-zebra.ts.net/";
+          };
+        };
+      };
+    };
+
+    haproxy = {
+      enable = true;
+      config = ''
+        global
+          daemon
+          maxconn 20
+
+        defaults
+          timeout connect 500s
+          timeout client 500s
+          timeout server 1h
+
+        listen gitsshd
+          bind *:${toString sshPort}
+          timeout client 1h
+          mode tcp
+          server git-isaiah isaiah.shire-zebra.ts.net:32222
+          server git-jeremiah jeremiah.shire-zebra.ts.net:32222
+          server git-zeke zeke.shire-zebra.ts.net:32222
+      '';
+    };
+  };
+
+  systemd.services = {
+    haproxy = {
+      after = [
+        "network-online.target"
+      ];
+      wants = [
+        "network-online.target"
+      ];
+    };
+  };
+
+  users.users.nginx.extraGroups = [ config.users.groups.anubis.name ];
+}

hosts/unstable/linode/matrix.nix

@@ -0,0 +1,70 @@
+# Registration of new users is disabled for the public, but I can create
+# them by the following commands:
+# nix run nixpkgs.matrix-synapse
+# register_new_matrix_user -k "B9EoPr2WV9hzwc7uL2Sx1JmvCeKDEOGCpB0uginQcQtEH4wzRtkSIdo7lltrjSQa" http://localhost:8448
+{ config, ... }:
+let
+  domain = "${config.networking.domain}";
+  fqdn = "matrix.${domain}";
+in
+{
+  greg.proxies."${fqdn}" = {
+    extraConfig = ''
+      error_log /var/log/nginx/debug.log debug;
+      proxy_ssl_verify off;
+      proxy_ssl_server_name on;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Forwarded-Ssl on;
+    '';
+    genAliases = false;
+    ssl = true;
+    target = "https://matrix.shire-zebra.ts.net";
+  };
+  services.nginx = {
+    virtualHosts = {
+      # Server the '.well-known' files to find the Matrix API server
+      "${domain}" = {
+        enableACME = true;
+        forceSSL = true;
+        # This is needed so that servers contacting hellings.com can find
+        # the actual application server at matrix.thehellings.com
+        locations."= /.well-known/matrix/server".extraConfig =
+          let
+            server = {
+              "m.server" = "${fqdn}:443";
+            };
+          in
+          ''
+            add_header Content-Type application/json;
+            return 200 '${builtins.toJSON server}';
+          '';
+
+        locations."= /.well-known/matrix/client".extraConfig =
+          let
+            client = {
+              "m.homeserver" = {
+                "base_url" = "https://${fqdn}";
+              };
+              "m.identity_server" = {
+                "base_url" = "https://vector.im";
+              };
+            };
+          in
+          ''
+            add_header Content-Type application/json;
+            add_header Access-Control-Allow-Origin *;
+            return 200 '${builtins.toJSON client}';
+          '';
+      };
+    };
+  };
+
+  # Open networking ports for the server
+  networking.firewall = {
+    enable = true;
+    allowedTCPPorts = [
+      80
+      443
+    ];
+  };
+}

hosts/unstable/linode/nextcloud.nix

@@ -0,0 +1,58 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  age.secrets.nextcloudadmin = {
+    file = ../../../secrets/nextcloudadmin.age;
+    owner = "nextcloud";
+  };
+
+  services.nextcloud = {
+    enable = true;
+    package = pkgs.nextcloud33;
+    appstoreEnable = true;
+    hostName = "next.${config.networking.domain}";
+    https = true;
+    config = {
+      adminpassFile = config.age.secrets.nextcloudadmin.path;
+      adminuser = "greg";
+      dbhost = "/run/postgresql";
+      dbtype = "pgsql";
+    };
+    settings = {
+      default_phone_region = "US";
+      overwriteprotocol = "https";
+    };
+  };
+
+  services.nginx.virtualHosts."next.thehellings.com" = {
+    forceSSL = true;
+    enableACME = true;
+  };
+
+  # Otherwise nginx errors looking for the nextcloud sock file
+  systemd.services.nginx.after = [ "nextcloud.service" ];
+
+  greg.backup.jobs.nextcloud-bkup = {
+    src = "/var/lib/nextcloud";
+    dest = "nextcloud-backup";
+    pre = lib.getExe (
+      pkgs.writeShellApplication {
+        name = "nextcloud-backup-pre";
+        runtimeInputs = [ config.services.nextcloud.occ ];
+        text = "nextcloud-occ maintenance:mode --on";
+      }
+    );
+    post = lib.getExe (
+      pkgs.writeShellApplication {
+        name = "nextcloud-backup-post";
+        runtimeInputs = [ config.services.nextcloud.occ ];
+        text = "nextcloud-occ maintenance:mode --off";
+      }
+    );
+  };
+}

hosts/unstable/linode/nginx.nix

@@ -0,0 +1,38 @@
+{ ... }:
+let
+  homepage = "127.0.0.1:30080";
+in
+{
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "greg.hellings@gmail.com";
+  };
+
+  services.nginx = {
+    enable = true;
+
+    clientMaxBodySize = "25000m"; # To help with uploading container images
+    # If there are recommended settings, let's use them!
+    recommendedGzipSettings = true;
+    recommendedOptimisation = true;
+    recommendedProxySettings = true;
+    recommendedTlsSettings = true;
+  };
+
+  # Actually serve the content from here
+  virtualisation.podman.enable = true;
+  virtualisation.oci-containers = {
+    backend = "podman";
+    containers."homepage" = {
+      image = "registry.thehellings.com:443/greg/homepage/gregs-homepage:latest";
+      ports = [ "${homepage}:80" ];
+    };
+  };
+  greg.proxies = {
+    "thehellings.com" = {
+      target = "http://${homepage}/";
+      ssl = true;
+      genAliases = false;
+    };
+  };
+}

hosts/unstable/linode/podman.nix

@@ -0,0 +1,13 @@
+{ pkgs, ... }:
+
+{
+  environment.systemPackages = with pkgs; [
+    podman-compose
+  ];
+
+  virtualisation.podman = {
+    enable = true;
+    dockerCompat = true;
+    dockerSocket.enable = true;
+  };
+}

hosts/unstable/linode/postgres.nix

@@ -0,0 +1,63 @@
+{
+  config,
+  pkgs,
+  pkgs',
+  ...
+}:
+
+{
+  environment.systemPackages = [ pkgs'.upgrade-pg-cluster ];
+
+  services.postgresql = {
+    enable = true;
+    package = pkgs.postgresql_15;
+    checkConfig = true;
+    ensureDatabases = [ "nextcloud" ];
+    #initialScript = pkgs.writeText "create-matrix-db.sql" ''
+    #	CREATE ROLE "matrix-synapse" WITH LOGIN;
+    #	CREATE DATABASE "synapse" WITH OWNER "matrix-synapse" TEMPLATE template0 LC_COLLATE = "C" LC_CTYPE = "C";
+    #	GRANT ALL PRIVILEGES ON DATABASE "synapse" TO "matrix-synapse";
+    #'';  # These are done manually in order to set the LC_COLLATE values properly
+    ensureUsers = [
+      {
+        name = "nextcloud";
+        ensureDBOwnership = true;
+      }
+    ];
+    settings = {
+      log_connections = true;
+      log_statement = "all";
+      logging_collector = true;
+      log_filename = "postgresql.log";
+    };
+    identMap = ''
+      root root postgres
+    '';
+  };
+
+  services.postgresqlBackup = {
+    enable = true;
+    databases = [ "nextcloud" ];
+  };
+
+  services.logrotate = {
+    enable = true;
+    settings = {
+      postgresBackup = {
+        enable = true;
+        files = "${config.services.postgresqlBackup.location}/*.gz";
+      };
+      postgresLog = {
+        enable = true;
+        files = "/var/lib/postgresql/*/log/*.log";
+        compress = true;
+        compresscmd = "${pkgs.xz}/bin/xz";
+      };
+    };
+  };
+
+  greg.backup.jobs.greg-postgresql-backup = {
+    src = config.services.postgresqlBackup.location;
+    dest = "linode-postgres";
+  };
+}

hosts/unstable/proxmoxtemplate/default.nix

@@ -0,0 +1,60 @@
+# Edit this configuration file to define what should be installed on
+# your system.  Help is available in the configuration.nix(5) man page
+# and in the NixOS manual (accessible by running ‘nixos-help’).
+
+{ pkgs, ... }:
+
+{
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+  ];
+
+  # Bootloader.
+  boot.loader = {
+    systemd-boot.enable = true;
+    efi.canTouchEfiVariables = true;
+  };
+
+  environment.systemPackages = with pkgs; [
+  ];
+
+  greg = {
+    home = true;
+    tailscale = {
+      enable = true;
+      tags = [ "home" ];
+    };
+  };
+
+  networking = {
+    hostName = "proxmoxtemplate"; # Define your hostname.
+    # defaultGateway = {
+    #   address = " 10.42.1.2";
+    #   interface = "enp6s18";
+    # };
+    # interfaces = {
+    #   enp6s18 = {
+    #     ipv4.addresses = [
+    #       {
+    #         address = "10.42.1.8";
+    #         prefixLength = 16;
+    #       }
+    #     ];
+    #   };
+    # };
+    nameservers = [ "10.42.1.5" ];
+  };
+
+  services.qemuGuest.enable = true;
+
+  system.stateVersion = "24.11"; # Did you read the comment?
+
+  # Define a user account. Don't forget to set a password with ‘passwd’.
+  users.users.greg = {
+    isNormalUser = true;
+    description = "Greg Hellings";
+    extraGroups = [ "wheel" ];
+    packages = with pkgs; [ ];
+  };
+}

hosts/unstable/proxmoxtemplate/hardware-configuration.nix

@@ -0,0 +1,50 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "uhci_hcd"
+    "ehci_pci"
+    "ahci"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/507251f1-efe7-448d-8de8-91ee582a9afb";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/7115-EFA6";
+    fsType = "vfat";
+    options = [
+      "fmask=0077"
+      "dmask=0077"
+    ];
+  };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp6s18.useDHCP = lib.mkDefault true;
+}

hosts/unstable/zeke/default.nix

@@ -32,10 +32,6 @@
       enable = true;
       tags = [ "home" ];
     };
-    vmdev = {
-      enable = true;
-      host = "vbox";
-    };
   };
 
   hardware = {
@@ -75,4 +71,10 @@
   users.users.greg.extraGroups = [
     "podman"
   ];
+
+  # virtualisation.virtualbox.host = {
+  #   enableExtensionPack = true;
+  #   headless = true;
+  #   enableWebService = true;
+  # };
 }

manifests/databases/kustomization.yaml

@@ -4,5 +4,6 @@ resources:
   - namespace.yaml
   - secrets.yaml
   - postgres-cluster.yaml
+  - postgres-pgadmin.yaml
   - postgres-matrix.yaml
   - ingress.yaml

manifests/databases/postgres-cluster.yaml

@@ -11,6 +11,13 @@ spec:
 
   managed:
     roles:
+      - name: pgadmin
+        ensure: present
+        comment: PG Admin user
+        login: true
+        superuser: true
+        passwordSecret:
+          name: postgres-user-pgadmin
       - name: matrix
         ensure: present
         comment: Matrix DB user

manifests/databases/postgres-pgadmin.yaml

@@ -0,0 +1,103 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-pgadmin
+data:
+  servers.json: |
+    {
+        "Servers": {
+          "1": {
+            "Name": "Postgres",
+            "Group": "Servers",
+            "Port": 5432,
+            "Username": "pgadmin",
+            "Host": "postgres-rw",
+            "SSLMode": "allow",
+            "MaintenanceDB": "postgres"
+          }
+        }
+    }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-pgadmin
+spec:
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: http
+  selector:
+    app: pgadmin
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: pgadmin
+spec:
+  serviceName: service-pgadmin
+  podManagementPolicy: Parallel
+  replicas: 1
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: pgadmin
+  template:
+    metadata:
+      labels:
+        app: pgadmin
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+        - name: pgadmin
+          image: "dpage/pgadmin4:9.3"
+          imagePullPolicy: Always
+          env:
+            - name: PGADMIN_DEFAULT_EMAIL
+              value: greg@thehellings.com
+            - name: PGADMIN_DEFAULT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-user-pgadmin
+                  key: password
+            - name: PGADMIN_SERVER_JSON_FILE
+              value: /config-pgadmin-vol/servers.json
+          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
+          volumeMounts:
+            - name: config-pgadmin-vol
+              mountPath: /config-pgadmin-vol/
+              readOnly: true
+            - name: pgadmin-data
+              mountPath: /var/lib/pgadmin
+      volumes:
+        - name: config-pgadmin-vol
+          configMap:
+            name: config-pgadmin
+  volumeClaimTemplates:
+    - metadata:
+        name: pgadmin-data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 3Gi
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress-pgadmin
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: service-pgadmin
+      port:
+        number: 80
+  tls:
+    - hosts:
+        - pgadmin

manifests/databases/secrets.yaml

@@ -34,6 +34,40 @@ spec:
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
+metadata:
+  name: postgres-user-pgadmin
+  namespace: db
+spec:
+  target:
+    name: postgres-user-pgadmin
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: f333d637-1667-499d-b9a0-b2e9012bd8b7
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: f333d637-1667-499d-b9a0-b2e9012bd8b7
+        property: password
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
 metadata:
   name: k3sbackup-externalsecret
   namespace: db

manifests/donetick/configmap.yaml

@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: donetick-config
+  namespace: donetick
+data:
+  # Value pulled from
+  # https://github.com/donetick/donetick/blob/main/config/selfhosted.yaml
+  selfhosted.yaml: |-
+    name: "selfhosted"
+    is_done_tick_dot_com: false
+    is_user_creation_disabled: false
+    telegram:
+      token: ""
+    pushover:
+      token: ""
+    database:
+      type: "sqlite"
+      migration: true
+      # these are only required for postgres
+      host: "secret"
+      port: 5432
+      user: "secret"
+      password: "secret"
+      name: "secret"
+    jwt:
+      secret: "This is really a secure JWT secret now!"
+      session_time: 168h
+      max_refresh: 168h
+    server:
+      port: 2021
+      read_timeout: 10s
+      write_timeout: 10s
+      rate_period: 60s
+      rate_limit: 300
+      cors_allow_origins:
+        - "http://localhost:5173"
+        - "http://localhost:7926"
+        # the below are required for the android app to work
+        - "https://localhost"
+        - "capacitor://localhost"
+      serve_frontend: true
+    logging:
+      level: "info"
+      encoding: "json"
+      development: false
+    scheduler_jobs:
+      due_job: 30m
+      overdue_job: 3h
+      pre_due_job: 3h
+    email:
+      host:
+      port:
+      key:
+      email:
+      appHost:
+    oauth2:
+      client_id:
+      client_secret:
+      auth_url:
+      token_url:
+      user_info_url:
+      redirect_url:
+      name:
+    # Real-time configuration
+    realtime:
+      enabled: true
+      sse_enabled: true
+      heartbeat_interval: 60s
+      connection_timeout: 120s
+      max_connections: 1000
+      max_connections_per_user: 5
+      event_queue_size: 2048
+      cleanup_interval: 2m
+      stale_threshold: 5m
+      enable_compression: true
+      enable_stats: true
+      allowed_origins:
+        - "*"

manifests/donetick/deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: donetick
+  namespace: donetick
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: donetick
+  template:
+    metadata:
+      labels:
+        app: donetick
+    spec:
+      containers:
+        - name: donetick
+          image: donetick/donetick
+          ports:
+            - containerPort: 2021
+              name: http
+          env:
+            - name: DT_ENV
+              value: "selfhosted"
+            - name: DT_SQLITE_PATH
+              value: "/data/donetick.db"
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: config
+          configMap:
+            name: donetick-config
+        - name: data
+          persistentVolumeClaim:
+            claimName: donetick-data

manifests/donetick/ingress.yaml

@@ -0,0 +1,15 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: donetick-tailscale
+  namespace: donetick
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: donetick
+      port:
+        number: 2021
+  tls:
+    - hosts:
+        - todo

manifests/donetick/kustomization.yaml

@@ -0,0 +1,9 @@
+namespace: donetick
+
+resources:
+  - namespace.yaml
+  - configmap.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

manifests/donetick/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: donetick

manifests/donetick/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: donetick-data
+  namespace: donetick
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

manifests/donetick/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: donetick
+  namespace: donetick
+spec:
+  selector:
+    app: donetick
+  ports:
+    - name: http
+      port: 2021
+      targetPort: 2021
+      protocol: TCP

manifests/gitea-runner/chart.yaml

@@ -0,0 +1,56 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: gitea
+spec:
+  interval: "24h"
+  url: https://dl.gitea.com/charts/
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: gitea-runner
+  namespace: gitea-runner
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: actions
+      version: "0.0.4"
+      sourceRef:
+        kind: HelmRepository
+        name: gitea
+      interval: "1h"
+  values:
+    rbac:
+      create: true
+    serviceAccount:
+      create: true
+    gitea:
+      instanceURL: https://src.thehellings.com
+      runnerToken:
+        existingSecret: gitea-runner
+        existingSecretKey: token
+    imagePullSecrets:
+      - name: image-pull-secrets
+    config:
+      runner:
+        labels:
+          # Ubuntu
+          - "ubuntu-22.04:docker://ubuntu:22.04"
+          - "ubuntu-24.04:docker://ubuntu:24.04"
+          - "ubuntu-24.10:docker://ubuntu:24.10"
+          # Fedora
+          - "fedora-41:docker://fedora:41"
+          - "fedora-42:docker://fedora:42"
+          # CentOS Stream
+          - "centos-stream-9:docker://quay.io/centos/centos:stream9"
+          - "centos-stream-10:docker://quay.io/centos/centos:stream10"
+          # Nix
+          - "nix:docker://nixos/nix:latest"
+          # ci-images (internal registry: src.thehellings.com/greg)
+          - "ci-builder:docker://src.thehellings.com/greg/builder:latest"
+          - "ci-vm-test:docker://src.thehellings.com/greg/vm-test:latest"
+          - "ci-sword:docker://src.thehellings.com/greg/sword-container-builder:latest"
+          - "ci-bitwarden:docker://src.thehellings.com/greg/bitwarden:latest"
+          - "ci-immich:docker://src.thehellings.com/greg/immich:latest"

manifests/gitea-runner/kustomization.yaml

@@ -0,0 +1,6 @@
+namespace: gitea-runner
+
+resources:
+  - namespace.yaml
+  - secrets.yaml
+  - chart.yaml

manifests/gitea-runner/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gitea-runner

manifests/gitea-runner/secrets.yaml

@@ -0,0 +1,18 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-runner
+  namespace: gitea-runner
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: bitwarden-login
+    kind: ClusterSecretStore
+  target:
+    name: gitea-runner
+    creationPolicy: Owner
+  data:
+    - secretKey: token
+      remoteRef:
+        key: 11419680-5338-4f19-bdd9-b422007046af
+        property: password

manifests/gitea/chart.yaml

@@ -15,7 +15,7 @@ spec:
   chart:
     spec:
       chart: gitea
-      version: "12.7.0"
+      version: "12.6.0"
       sourceRef:
         kind: HelmRepository
         name: gitea-repository
@@ -35,7 +35,7 @@ spec:
       storageClass: longhorn-default
 
     image:
-      tag: "1.27.1"
+      tag: "1.26.2"
 
     replicaCount: 1
 
@@ -67,7 +67,7 @@ spec:
           RUN_MODE: dev
         server:
           DOMAIN: "shire-zebra.ts.net"
-          ROOT_URL: "https://git.k3s.thehellings.lan"
+          ROOT_URL: "https://gitea.shire-zebra.ts.net"
           SSH_PORT: "2222"
         database:
           DB_TYPE: postgres
@@ -85,15 +85,15 @@ spec:
           DISABLE_REGISTRATION: "true"
         storage:
           STORAGE_TYPE: minio
-          MINIO_ENDPOINT: "nas1.shire-zebra.ts.net:30188"
+          MINIO_ENDPOINT: "nas1.shire-zebra.ts.net:9000"
           MINIO_BUCKET: gitea
-          MINIO_LOCATION: garage
+          MINIO_LOCATION: us-east-1
           # MINIO_ACCESS_KEY_ID: ""
           # MINIO_SECRET_ACCESS_KEY: ""
           MINIO_USE_SSL: "false"
           MINIO_INSECURE_SKIP_VERIFY: "true"
-        security:
-          ALLOWED_HOST_LIST: loopback,private,*.shire-zebra.ts.net,*.nebula.thehellings.com,*.thehellings.lan
+        webhook:
+          ALLOWED_HOST_LIST: loopback,private,*.shire-zebra.ts.net
 
       metrics:
         enabled: false
@@ -102,8 +102,8 @@ spec:
 
     persistence:
       enabled: true
-      create: false
-      claimName: gitea-new
+      storageClass: longhorn-default
+      size: "50Gi"
 
     # I will manage my Postgres externally
     postgresql:

manifests/gitea/dump-cronjob.yaml

@@ -18,12 +18,12 @@ spec:
           volumes:
             - name: gitea-data
               persistentVolumeClaim:
-                claimName: gitea-new
+                claimName: gitea-shared-storage
             - name: dump-staging
               emptyDir: {}
           initContainers:
             - name: gitea-dump
-              image: "gitea/gitea:1.27.1"
+              image: "gitea/gitea:1.25.4"
               command:
                 - /bin/sh
                 - "-c"
@@ -51,12 +51,12 @@ spec:
                 - |
                   set -e
                   # Configure mc alias for MinIO
-                  mc alias set nas1 http://nas1.shire-zebra.ts.net:30188 \
+                  mc alias set nas1 http://nas1.shire-zebra.ts.net:9000 \
                     "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
                   # Upload dump to backup-gitea bucket
                   DUMP_FILE=$(ls /dump-staging/gitea-dump-*.zip | head -1)
-                  mc cp "${DUMP_FILE}" "nas1/gitea-backup/$(basename ${DUMP_FILE})"
-                  echo "Uploaded $(basename ${DUMP_FILE}) to gitea-backup"
+                  mc cp "${DUMP_FILE}" "nas1/backup-gitea/$(basename ${DUMP_FILE})"
+                  echo "Uploaded $(basename ${DUMP_FILE}) to backup-gitea"
                   # Set 30-day lifecycle on the bucket (idempotent)
                   mc ilm rule add --expire-days 30 nas1/backup-gitea 2>/dev/null || true
               volumeMounts:
@@ -69,10 +69,10 @@ spec:
                 - name: MINIO_ACCESS_KEY
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-backup
+                      name: gitea-config
                       key: minio_key
                 - name: MINIO_SECRET_KEY
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-backup
+                      name: gitea-config
                       key: minio_secret

manifests/gitea/ingress.yaml

@@ -12,20 +12,3 @@ spec:
   tls:
     - hosts:
         - gitea
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gitea-direct
-spec:
-  rules:
-    - host: git.k3s.thehellings.lan
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: gitea-release-http
-                port:
-                  name: http

manifests/gitea/secrets.yaml

@@ -1,32 +1,5 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
-metadata:
-  name: gitea-backup
-spec:
-  target:
-    name: gitea-backup
-    deletionPolicy: Delete
-    template:
-      type: Opaque
-      data:
-        minio_key: "{{ .minio_key }}"
-        minio_secret: "{{ .minio_secret }}"
-  secretStoreRef:
-    name: bitwarden-login
-    kind: ClusterSecretStore
-  data:
-    # MinIO credentials
-    - secretKey: minio_key
-      remoteRef:
-        key: dfb2f0c8-110d-4e96-83a7-b49c001c0897
-        property: username
-    - secretKey: minio_secret
-      remoteRef:
-        key: dfb2f0c8-110d-4e96-83a7-b49c001c0897
-        property: password
----
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
 metadata:
   name: gitea-config
 spec:
@@ -48,14 +21,23 @@ spec:
     name: bitwarden-login
     kind: ClusterSecretStore
   data:
+    # MinIO credentials
+    - secretKey: minio_key
+      remoteRef:
+        key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d
+        property: username
+    - secretKey: minio_secret
+      remoteRef:
+        key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d
+        property: password
     # MinIO credentials for NAS1
     - secretKey: minio_nas1_key
       remoteRef:
-        key: 33e8e4e0-eb90-484c-9ec9-b3a8018077a3
+        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
         property: username
     - secretKey: minio_nas1_secret
       remoteRef:
-        key: 33e8e4e0-eb90-484c-9ec9-b3a8018077a3
+        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
         property: password
     # Postgres credentials
     - secretKey: dbuser

manifests/helm/longhorn.yaml

@@ -27,7 +27,7 @@ spec:
   chart:
     spec:
       chart: longhorn
-      version: "1.11.3"
+      version: "1.11.2"
       sourceRef:
         kind: HelmRepository
         name: longhorn
@@ -141,10 +141,6 @@ spec:
                   number: 80
     - <<: *host
       host: longhorn.kubernetes
-    - <<: *host
-      host: longhorn.k3s.nebula.thehellings.com
-    - <<: *host
-      host: longhorn.k3s.thehellings.lan
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass

manifests/helm/traefik.yaml

@@ -10,16 +10,6 @@ spec:
       - "--api.dashboard=true"
       - "--api.insecure=true"
       - "--log.level=DEBUG"
-      # Access logging: gives per-request visibility (client IP, host,
-      # path, bytes, duration) for every ingress route Traefik terminates
-      # (git.k3s.thehellings.lan, matrix.k3s.thehellings.lan, immich, etc).
-      # This is the layer HAProxy on linode forwards :80 traffic to, so
-      # having request-level logs here is essential for tracing bandwidth
-      # spikes back to a specific host/path/client rather than just a
-      # backend-level byte count.
-      - "--accesslog=true"
-      - "--accesslog.format=json"
-      - "--accesslog.fields.headers.defaultmode=keep"
     ports:
       postgres:
         expose:
@@ -30,4 +20,3 @@ spec:
       traefik:
         expose:
           default: true
-

manifests/immich/backup.yaml

@@ -33,24 +33,24 @@ spec:
         access-key: "{{ .minio_key }}"
         secret-key: "{{ .minio_secret }}"
         rclone.conf: |
-          [garage]
+          [nas1minio]
           type = s3
           provider = Minio
-          endpoint = http://nas1.shire-zebra.ts.net:30188
+          endpoint = http://nas1.shire-zebra.ts.net:9000
           access_key_id = {{ .minio_key }}
           secret_access_key = {{ .minio_secret }}
-          region = garage
+          region = us-east-1
   secretStoreRef:
     name: bitwarden-login
     kind: ClusterSecretStore
   data:
     - secretKey: minio_key
       remoteRef:
-        key: 8fce2750-aa62-4892-b90c-b49c001f494b
+        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
         property: username
     - secretKey: minio_secret
       remoteRef:
-        key: 8fce2750-aa62-4892-b90c-b49c001f494b
+        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
         property: password
 ---
 apiVersion: v1
@@ -128,7 +128,7 @@ spec:
                     --progress \
                     --transfers 4 \
                     --checkers 8 \
-                    /staging garage:immich
+                    /staging nas1minio:immich
               volumeMounts:
                 - name: staging
                   mountPath: /staging

manifests/immich/chart.yaml

@@ -32,7 +32,7 @@ spec:
         containers:
           main:
             image:
-              tag: v3.1.0
+              tag: v2.7.5
             env:
               DB_HOSTNAME: immich-rw
               DB_DATABASE_NAME: immich

manifests/immich/ingress.yaml

@@ -22,10 +22,6 @@ spec:
                 name: immich-server
                 port:
                   name: http
-    - <<: *host
-      host: immich.k3s.nebula.thehellings.com
-    - <<: *host
-      host: immich.k3s.thehellings.lan
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress

manifests/kustomization.yaml

@@ -10,4 +10,7 @@ resources:
   - immich
   - monitoring
   - pinchflat
+  - smokeping
+  - uptimekuma
+  - donetick
   - gitea

manifests/matrix/ingress.yaml

@@ -13,20 +13,3 @@ spec:
   tls:
     - hosts:
         - matrix
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: matrix-direct
-spec:
-  rules:
-    - host: matrix.k3s.thehellings.lan
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: dendrite
-                port:
-                  number: 8008

manifests/monitoring/config.yaml

@@ -51,8 +51,8 @@ data:
         static_configs:
           - targets:
               - "10.42.0.3"    # OpenWRT access point
-              - "10.42.0.4"    # pve1 (Proxmox)
-              - "10.42.1.1"    # UDM gateway (Ubiquiti)
+              - "10.42.0.4"    # Joel (Proxmox)
+              - "10.42.1.1"    # pve1 (Proxmox)
               - "10.42.1.4"    # chronicles (Synology NAS)
               - "10.42.1.14"   # nas1 (TrueNAS)
               - "10.42.2.57"   # odoo

manifests/smokeping/deployment.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: smokeping
+  labels:
+    app: smokeping
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: smokeping
+  template:
+    metadata:
+      labels:
+        app: smokeping
+    spec:
+      containers:
+        - name: smokeping
+          image: docker.io/linuxserver/smokeping:2.9.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
+          volumeMounts:
+            - name: config
+              mountPath: /config
+            - name: data
+              mountPath: /data
+          env:
+            - name: PUID
+              value: "1000"
+            - name: PGID
+              value: "1000"
+            - name: TZ
+              value: "America/Chicago"
+            #- name: MASTER_URL
+            #  value: "https://ping.shire-zebra.ts.net"
+            # SHARED_SECRET if you want to run a cluster
+            # CACHE_DIR if you need to explicitly state that
+      restartPolicy: Always
+      volumes:
+        - name: config
+          persistentVolumeClaim:
+            claimName: smokeping-config
+        - name: data
+          persistentVolumeClaim:
+            claimName: smokeping-data

manifests/smokeping/ingress.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: smokeping-tailscale
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: smokeping
+      port:
+        name: http
+  tls:
+    - hosts:
+        - ping

manifests/smokeping/kustomization.yaml

@@ -0,0 +1,8 @@
+namespace: smokeping
+
+resources:
+  - namespace.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

manifests/smokeping/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: smokeping

manifests/smokeping/pvc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: smokeping-config
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-default
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: smokeping-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-default
+  resources:
+    requests:
+      storage: 25Gi

manifests/smokeping/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: smokeping
+  labels:
+    app: smokeping
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: smokeping

manifests/uptimekuma/chart.yaml

@@ -0,0 +1,38 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: uptime-kuma
+  namespace: uptime-kuma
+spec:
+  interval: "24h"
+  url: "https://helm.irsigler.cloud"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: uptime-kuma
+  namespace: uptime-kuma
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: uptime-kuma
+      sourceRef:
+        kind: HelmRepository
+        name: uptime-kuma
+      interval: "1h"
+  dependsOn:
+    - name: longhorn
+      namespace: longhorn-system
+    - name: mariadb-cluster
+      namespace: mariadb-operator
+  values:
+    volume:
+      storageClassName: longhorn-default
+    image:
+      tag: "2.0.2"
+    externalDatabase:
+      enabled: true
+      hostname: mariadb-cluster.mariadb-operator.svc.cluster.local
+      database: uptimekuma
+      existingSecret: uptimekuma-mariadb-password

manifests/uptimekuma/ingress.yaml

@@ -0,0 +1,15 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: uptime-kuma-tailscale
+  namespace: uptime-kuma
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: uptime-kuma
+      port:
+        number: 3001
+  tls:
+    - hosts:
+        - kuma

manifests/uptimekuma/kustomization.yaml

@@ -0,0 +1,7 @@
+namespace: uptimekuma
+
+resources:
+  - namespace.yaml
+  - secrets.yaml
+  - chart.yaml
+  - ingress.yaml

manifests/uptimekuma/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: uptimekuma

manifests/uptimekuma/secrets.yaml

@@ -0,0 +1,27 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: uptimekuma-mariadb-password
+spec:
+  target:
+    name: uptimekuma-mariadb-password
+    deletionPolicy: Delete
+    template:
+      type: kubernetes.io/basic-auth
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  secretStoreRef:
+    name: bitwarden-login
+    kind: ClusterSecretStore
+  data:
+    - secretKey: username
+      remoteRef:
+        key: 4df95656-9f9c-4916-8e34-b3a200376365
+        property: username
+    - secretKey: password
+      remoteRef:
+        key: 4df95656-9f9c-4916-8e34-b3a200376365
+        property: password

modules/hm/gui/bookmarks.nix

@@ -230,7 +230,7 @@
             bookmarks = [
               {
                 name = "PVE1";
-                url = "https://10.42.0.4:8006/";
+                url = "https://10.42.1.1:8006/";
               }
               {
                 name = "Jeremiah";
@@ -253,6 +253,10 @@
                 name = "Longhorn";
                 url = "http://longhorn.shire-zebra.ts.net";
               }
+              {
+                name = "PGAdmin4";
+                url = "http://pgadmin.shire-zebra.ts.net/";
+              }
             ];
           }
           {

modules/nix-conf.nix

@@ -62,12 +62,12 @@ in
             if cfg.cache then
               [
                 #"http://chronicles.shire-zebra.ts.net:9000/binary-cache/"
-                "http://niks3.nas1.shire-zebra.ts.net:30189/"
+                "http://nas1.shire-zebra.ts.net:9000/niks3"
                 #"http://nas1.shire-zebra.ts.net:8080/default"
               ]
             else
               [
-                "http://niks3.nas1.thehellings.lan:30189/"
+                "http://nas1.thehellings.lan:8080/default"
               ]
           )
           ++ [

modules/nixos/ceph.nix

@@ -0,0 +1,927 @@
+# This is a good source for a Ceph dealio
+# https://gist.github.com0/nh2/13425a1f18b4c1ce82edb63c10b163c9
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  cfg = config.services.ceph-benaco;
+  commaSep = builtins.concatStringsSep ",";
+
+  ensureUnitExists =
+    c': name:
+    let
+    in
+    #unitName = (builtins.elemAt (builtins.split "\\." name) 0);
+    if c'.systemd.services ? unitName then name else name; # "Unable to locate ${name} at ${commaSep (builtins.attrNames c')}";
+in
+
+{
+
+  ###### interface
+
+  options = {
+
+    services.ceph-benaco = {
+
+      enable = mkEnableOption "Ceph distributed filesystem";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.ceph;
+        defaultText = literalExpression "pkgs.ceph-benaco";
+        description = "Ceph package to use.";
+      };
+
+      fsid = mkOption {
+        type = types.str;
+        description = "Unique cluster identifier.";
+      };
+
+      clusterName = mkOption {
+        type = types.str;
+        description = "Cluster name.";
+        default = "ceph";
+      };
+
+      initialMonitors = mkOption {
+        type = types.listOf (
+          types.submodule {
+            options = {
+              hostname = mkOption {
+                type = types.str;
+                description = "Initial monitor hostname.";
+              };
+
+              ipAddress = mkOption {
+                type = types.str;
+                description = "Initial monitor IP address.";
+              };
+            };
+          }
+        );
+        description = "Initial monitors.";
+      };
+
+      mdsNodes = mkOption {
+        type = types.listOf (
+          types.submodule {
+            options = {
+              hostname = mkOption {
+                type = types.str;
+                description = "MDS hostname.";
+              };
+
+              ipAddress = mkOption {
+                type = types.str;
+                description = "MDS IP address.";
+              };
+            };
+          }
+        );
+        description = "MDS nodes.";
+      };
+
+      publicNetworks = mkOption {
+        type = types.listOf types.str;
+        description = "Public network(s) of the cluster.";
+      };
+
+      clusterNetworks = mkOption {
+        type = types.listOf types.str;
+        description = "Cluster backend networks for OSD sync";
+      };
+
+      adminKeyring = mkOption {
+        type = types.path;
+        description = "Ceph admin keyring to install on the machine.";
+      };
+
+      monitor = {
+        enable = mkEnableOption "Activate a Ceph monitor on this machine.";
+
+        initialKeyring = mkOption {
+          type = types.path;
+          description = "Keyring file to use when initializing a new monitor";
+          example = "/path/to/ceph.mon.keyring";
+        };
+
+        nodeName = mkOption {
+          type = types.str;
+          description = "Ceph monitor node name.";
+          example = "node1";
+        };
+
+        bindAddr = mkOption {
+          type = types.str;
+          description = "IP address that the OSDs shall bind to.";
+          example = "10.0.0.1";
+        };
+
+        advertisedPublicAddr = mkOption {
+          type = types.str;
+          description = "IP address that the monitor shall advertise.";
+          example = "10.0.0.1";
+        };
+      };
+
+      manager = {
+        enable = mkEnableOption "Activate a Ceph manager on this machine.";
+
+        nodeName = mkOption {
+          type = types.str;
+          description = "Ceph manager node name.";
+          example = "node1";
+        };
+      };
+
+      osdBindAddr = mkOption {
+        type = types.str;
+        description = "IP address that the OSDs shall bind to.";
+        example = "10.0.0.1";
+      };
+
+      osdAdvertisedPublicAddr = mkOption {
+        type = types.str;
+        description = "IP address that the OSDs shall advertise.";
+        example = "10.0.0.1";
+      };
+
+      osds = mkOption {
+        default = { };
+        example = {
+          osd1 = {
+            enable = true;
+            bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
+            id = 1;
+            uuid = "11111111-1111-1111-1111-111111111111";
+            blockDevice = "/dev/sdb";
+            blockDeviceUdevRuleMatcher = ''KERNEL=="sdb"'';
+            clusterAddress = "10.1.0.1";
+          };
+          osd2 = {
+            enable = true;
+            bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
+            id = 2;
+            uuid = "22222222-2222-2222-2222-222222222222";
+            blockDevice = "/dev/sdc";
+            blockDeviceUdevRuleMatcher = ''KERNEL=="sdc"'';
+            clusterAddress = "10.1.0.2";
+          };
+        };
+        description = ''
+          This option allows you to define multiple Ceph OSDs.
+          A common idiom is to use one OSD per physical hard drive.
+
+          Note that the OSD names given as attributes of this key
+          are NOT what ceph calls OSD IDs (instead, those are defined
+          by the 'services.ceph-benaco.osds.*.id' fields).
+          Instead, the name is an identifier local and unique to the
+          current machine only, used only to name the systemd service
+          for that OSD.
+        '';
+        type = types.attrsOf (
+          types.submodule {
+            options = {
+
+              enable = mkEnableOption "Activate a Ceph OSD on this machine.";
+
+              bootstrapKeyring = mkOption {
+                type = types.path;
+                description = "Ceph OSD bootstrap keyring.";
+                example = "/path/to/ceph.client.bootstrap-osd.keyring";
+              };
+
+              id = mkOption {
+                type = types.int;
+                description = "The ID of this OSD. Must be unique in the Ceph cluster.";
+                example = 1;
+              };
+
+              uuid = mkOption {
+                type = types.str;
+                description = "The UUID of this OSD. Must be unique in the Ceph cluster.";
+                example = "abcdef12-abcd-1234-abcd-1234567890ab";
+              };
+
+              systemdExtraRequiresAfter = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Add the specified systemd units to the "requires" and "after"
+                  lists of the systemd service of this OSD.
+
+                  Useful, for example, to decrypt the underlying block devices with LUKS first.
+
+                  NixOS modules allow override those lists from outside, but for that
+                  the names of the systemd services for the OSDs need to be known;
+                  this option is a convenience to not have to know them from outside.
+                '';
+                example = "decrypt-my-disk.service";
+              };
+
+              skipZap = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Whether to skip the zapping of the the OSD device on initial OSD
+                  installation.
+
+                  Skipping is needed because <command>ceph-volume</command> cannot
+                  zap device-mapper devices:
+                  <link xlink:href="https://tracker.ceph.com/issues/24504" />
+
+                  In that case you need to wipe the device manually.
+
+                  In the common case of placing the OSD on a cryptsetup LUKS device
+                  (which is a device-mapper device), re-creating the encryption
+                  from scratch with a new key zaps anything anyway, in which case
+                  zapping can be skipped here.
+                '';
+              };
+
+              blockDevice = mkOption {
+                type = types.str;
+                description = "The block device used to store the OSD.";
+                example = "/dev/sdb";
+              };
+
+              blockDeviceUdevRuleMatcher = mkOption {
+                type = types.str;
+                description = ''
+                  An udev rule matcher matching the block device used to store the OSD.
+                  Will be spliced into the udev rule that is
+                  used to set access permissions to the ceph user via an udev rule.
+
+                  This is a matcher instead of just a device name to allow flexibility:
+                  Normal disks can be easily matched with <code>KERNEL=="sda1"</code>, but
+                  device-mapper may not; for example, decrypted cryptsetup LUKS devices
+                  have a less useful <code>KERNEL=="dm-4"</code> and may better be matched
+                  using <code>ENV{DM_NAME}=="mydisk-decrypted"</code>.
+                '';
+                example = ''KERNEL=="sdb"'';
+              };
+
+              dbBlockDevice = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = ''
+                  The block device used to store the OSD's BlueStore DB device.
+
+                  Put this on a faster device than <option>blockDevice</option> to improve performance.
+
+                  See <link xlink:href="http://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/" />
+                  for details.
+                '';
+                example = "/dev/sdc";
+              };
+
+              dbBlockDeviceUdevRuleMatcher = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = ''
+                  Like <option>blockDeviceUdevRuleMatcher</option> but for the
+                  <option>dbBlockDevice</option>.
+                '';
+                example = ''KERNEL=="sdc"'';
+              };
+
+              clusterAddress = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = ''
+                  The IP address on the dedicated cluster network that
+                  is used by the backend communication for OSD communication.
+                '';
+                example = "10.1.0.1f";
+              };
+
+            };
+          }
+        );
+      };
+
+      mds = {
+        enable = mkEnableOption "Activate a Ceph MDS on this machine.";
+
+        nodeName = mkOption {
+          type = types.str;
+          description = "Ceph MDS node name.";
+          example = "node1";
+        };
+
+        listenAddr = mkOption {
+          type = types.str;
+          description = "IP address that the MDS shall advertise.";
+          example = "10.0.0.1";
+        };
+      };
+
+      extraConfig = mkOption {
+        type = types.str;
+        default = "";
+        description = ''
+          Additional ceph.conf settings.
+
+          See the sample file for inspiration:
+          <link xlink:href="https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf" />
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+
+  config =
+    let
+      monDir = "/var/lib/ceph/mon/${cfg.clusterName}-${cfg.monitor.nodeName}";
+      mgrDir = "/var/lib/ceph/mgr/${cfg.clusterName}-${cfg.manager.nodeName}";
+      mdsDir = "/var/lib/ceph/mds/${cfg.clusterName}-${cfg.mds.nodeName}";
+
+      # File permissions for things that are on locations wiped at start
+      # (e.g. /run or its /var/run symlink).
+      ensureTransientCephDirs = ''
+        install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
+      '';
+
+      # File permissions from cluster deployed with ceph-deploy.
+      ensureCephDirs = ''
+        install -m 3770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/log/ceph
+        install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
+        install -m 750 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph
+        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mon
+        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mgr
+        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/osd
+      '';
+
+      # Utilities called by Ceph device health scraping, see:
+      # https://docs.ceph.com/en/latest/rados/operations/devices/#enabling-monitoring
+      # As per https://github.com/ceph/ceph-container/pull/1490/commits/c49e821599965ae92a88b2c78077ee03c4405895,
+      # both the OSDs and the `mon` need this.
+      # Ceph calls these utilities with `sudo`. That requires sudoers entries.
+      # Sudoers entries require absolute path; that exact (nix store) path needs to
+      # be used by Ceph, so it needs to be given to the systemd unit via `path`.
+      # This is why we pair each `sudoersExtraRule` with the `package` to put onto
+      # that `path`.
+      #
+      # Entries are based on:
+      # https://github.com/ceph/ceph/blob/a2f5a3c1dbfa4dce41e25da4f029a8fdb8c8d864/sudoers.d/ceph-smartctl
+      cephMonitoringSudoersCommandsAndPackages = [
+        {
+          package = pkgs.smartmontools;
+          sudoersExtraRule = {
+            # entry for `security.sudo.extraRules`
+            users = [ config.users.users.ceph.name ];
+            commands = [
+              {
+                command = "${lib.getBin pkgs.smartmontools}/bin/smartctl -x --json=o /dev/*";
+                options = [ "NOPASSWD" ];
+              }
+            ];
+          };
+        }
+        {
+          package = pkgs.nvme-cli;
+          sudoersExtraRule = {
+            # entry for `security.sudo.extraRules`
+            users = [ config.users.users.ceph.name ];
+            commands = [
+              {
+                command = "${lib.getBin pkgs.nvme-cli}/bin/nvme * smart-log-add --json /dev/*";
+                options = [ "NOPASSWD" ];
+              }
+            ];
+          };
+        }
+      ];
+
+      cephDeviceHealthMonitoringPathsOrPackages =
+        with pkgs;
+        [
+          # Contains `sudo`. Ceph wraps this around the other health check programs.
+          # Cannot use `pkgs.sudo` because that one is not SUID, see:
+          # https://discourse.nixos.org/t/sudo-uid-issues/9133
+          "/run/wrappers" # `systemd.services.<name>.path` adds the `bin/` subdir of this
+        ]
+        ++ map ({ package, ... }: package) cephMonitoringSudoersCommandsAndPackages;
+
+      # Unused localOsdServiceName in the following line
+      # deadnix: skip
+      makeCephOsdSetupSystemdService =
+        _localOsdServiceName: osdConfig:
+        let
+          osdExistenceFile = "/var/lib/ceph/osd/.${toString osdConfig.id}.${osdConfig.uuid}.nix-existence";
+        in
+        mkIf osdConfig.enable {
+          description = "Initialize Ceph OSD";
+
+          requires = osdConfig.systemdExtraRequiresAfter;
+          after = osdConfig.systemdExtraRequiresAfter;
+
+          path = with pkgs; [
+            # The following are currently missing in Ceph's wrapping, see https://github.com/NixOS/nixpkgs/issues/147801#issue-1065600852
+            util-linux # for `lsblk`
+            lvm2 # for `lvs`
+          ];
+
+          # TODO Use `udevadm trigger --settle` instead of the separate `udevadm settle`
+          #      once that feature is available to us with systemd >= 238;
+          #      see https://github.com/systemd/systemd/commit/792cc203a67edb201073351f5c766fce3d5eab45
+          preStart = ''
+            set -x
+            ${ensureCephDirs}
+            install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/bootstrap-osd
+            # `install` is not atomic, see
+            # https://lists.gnu.org/archive/html/bug-coreutils/2010-02/msg00243.html
+            # so use `mktemp` + `mv` to make it atomic.
+            TMPFILE=$(mktemp --tmpdir=/var/lib/ceph/bootstrap-osd/)
+            install -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} ${osdConfig.bootstrapKeyring} "$TMPFILE"
+            mv "$TMPFILE" /var/lib/ceph/bootstrap-osd/ceph.keyring
+
+            # Trigger udev rules for permissions of block devices and wait for them to settle.
+            udevadm trigger --name-match=${osdConfig.blockDevice}
+          ''
+          + lib.optionalString (osdConfig.dbBlockDevice != null) ''
+            udevadm trigger --name-match=${osdConfig.dbBlockDevice}
+          ''
+          + ''
+            udevadm settle
+          ''
+          + (optionalString (!osdConfig.skipZap) (
+            ''
+              # Zap OSD block devices, otherwise `ceph-osd` below will try to fsck if there's some old
+              # ceph data on the block device (see https://tracker.ceph.com/issues/24099).
+              ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.blockDevice}
+            ''
+            + lib.optionalString (osdConfig.dbBlockDevice != null) ''
+              ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.dbBlockDevice}
+            ''
+          ));
+
+          script = ''
+            set -euo pipefail
+            set -x
+            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
+            do
+              sleep 1
+            done
+
+            OSD_SECRET=$(${cfg.package}/bin/ceph-authtool --gen-print-key)
+            echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
+              ${cfg.package}/bin/ceph --cluster ${cfg.clusterName} osd new ${osdConfig.uuid} ${toString osdConfig.id} -i - \
+              -n client.bootstrap-osd -k ${osdConfig.bootstrapKeyring}
+            mkdir -p /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}
+
+            ln -s ${osdConfig.blockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block
+          ''
+          + lib.optionalString (osdConfig.dbBlockDevice != null) ''
+            ln -s ${osdConfig.dbBlockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block.db
+          ''
+          + ''
+
+            ${cfg.package}/bin/ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${toString osdConfig.id}/keyring \
+              --name osd.${toString osdConfig.id} --add-key $OSD_SECRET
+
+            ${cfg.package}/bin/ceph-osd -i ${toString osdConfig.id} --mkfs --osd-uuid ${osdConfig.uuid} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} --osd-objectstore bluestore
+            touch ${osdExistenceFile}
+          '';
+
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            PermissionsStartOnly = true; # only run the script as ceph, preStart as root
+            User = config.users.users.ceph.name;
+            Group = config.users.groups.ceph.name;
+          };
+          unitConfig = {
+            ConditionPathExists = "!${osdExistenceFile}";
+          };
+        };
+
+      makeCephOsdSystemdService =
+        localOsdServiceName: osdConfig:
+        mkIf osdConfig.enable {
+          description = "Ceph OSD";
+
+          # Note we do not have to add `osdConfig.systemdExtraRequiresAfter` here because
+          # that's already a dependency of our dependency `ceph-osd-setup-*`.
+          requires = [ (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service") ];
+          requiredBy = [ "multi-user.target" ];
+          after = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+            (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service")
+          ];
+          wants = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+          ];
+
+          path = [
+            # TODO: use wrapProgram in the ceph package for this in the future
+            pkgs.getopt
+          ]
+          ++ cephDeviceHealthMonitoringPathsOrPackages;
+
+          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
+
+          preStart = ''
+            ${ensureTransientCephDirs}
+            ${lib.getLib cfg.package}/libexec/ceph/ceph-osd-prestart.sh --cluster ${cfg.clusterName} --id ${toString osdConfig.id}
+          '';
+
+          serviceConfig =
+            let
+              clusterIpArg = lib.optionalString (
+                osdConfig.clusterAddress != null
+              ) "--cluster_addr=${osdConfig.clusterAddress}";
+            in
+            {
+              LimitNOFILE = "1048576";
+              LimitNPROC = "1048576";
+
+              ExecStart = ''
+                ${cfg.package}/bin/ceph-osd -f --cluster ${cfg.clusterName} --id ${toString osdConfig.id} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.osdBindAddr}" "--public_addr=${cfg.osdAdvertisedPublicAddr}" "${clusterIpArg}"
+              '';
+              ExecReload = ''
+                ${pkgs.coreutils}/bin/kill -HUP $MAINPID
+              '';
+              Restart = "on-failure";
+              ProtectHome = "true";
+              ProtectSystem = "full";
+              PrivateTmp = "true";
+              TasksMax = "infinity";
+              # StartLimitBurst="3";
+            };
+          # startLimitIntervalSec = 30 * 60;
+        };
+
+    in
+    mkIf cfg.enable {
+      environment.systemPackages = [ cfg.package ];
+
+      networking.firewall = {
+        allowedTCPPorts = [
+          # Ceph outside of VPN because it is very data heavy and causes packet loss.
+          # We enable msgr-v2 only because that allows its own on-wire encryption.
+          3300 # ceph msgr-v2
+        ];
+        allowedTCPPortRanges = [
+          {
+            from = 6800;
+            to = 7300;
+          } # https://docs.ceph.com/en/pacific/rados/configuration/network-config-ref/
+        ];
+      };
+
+      # Reminder of how `ceph.conf` works:
+      #
+      # * Ceph upstream docs now recommend to use underscores instead of spaces.
+      # * Options in more specific sections like `[mon]` override those in less
+      #   specific sections like `[global]`. But all options can be written into all sections,
+      #   and an option has the same name, no matter in which section it is written.
+      #   Thus, put options in `[global]`, and only use a diffent section
+      #   if you want to override an option you've set in `global`.
+      #
+      # Sample: https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf
+      environment.etc."ceph/${cfg.clusterName}.conf".text = ''
+        [global]
+        fsid = ${cfg.fsid}
+        mon_initial_members = ${commaSep (map (mon: mon.hostname) cfg.initialMonitors)}
+        mon_host = ${commaSep (map (mon: mon.ipAddress) cfg.initialMonitors)}
+
+        # Ceph clusters go into WARN health mode, until
+        # the following setting is made strict by setting it to `false`:
+        # See: https://docs.ceph.com/en/latest/security/CVE-2021-20288/#recommendations
+        # As of writing, this setting is not documented outside of the CVE note :(
+        #
+        # While for new clusters the warning no longer seems to appear, it still
+        # appears in our existing clusters unless this option is set, see:
+        # https://tracker.ceph.com/issues/53751#note-7
+        auth_allow_insecure_global_id_reclaim = false
+
+        # Disable dirfrag prefetch on MDS restart to prevent out-of-memory after
+        # many files were opened.
+        # Note this option has no effect on Ceph < 15, because it doesn't exist there.
+        # TODO: Remove this once we're on a Ceph version that includes this default,
+        #       see https://github.com/ceph/ceph/pull/44667.
+        #       This is assuming that the commit fixes existing clusters, see
+        #       https://github.com/ceph/ceph/pull/44667#issuecomment-1036103397
+        #       If it doesn't this can only be removed once we have no existing
+        #       cluster with the old default.
+        mds_oft_prefetch_dirfrags = false
+
+        # Disable sleep between HDD recovery operations, otherwise recovery
+        # will take forever when small objects (e.g. CephFS files) are on HDD.
+        # See https://tracker.ceph.com/issues/23595#note-12
+        osd_recovery_sleep_hdd = 0.0
+
+        # Increase scrub intervals by 4x.
+        # Since we store many small files on HDD, and scrubbing apparently
+        # iterates over all objects
+        # we have no chance to scrub at the default intervals.
+        #
+        # (This was written when we had 400M files across 30 HDDs.)
+        # Change this back once we have reduced our number of files per disk.
+        osd_scrub_min_interval = 345600
+        osd_scrub_max_interval = 2419200
+        osd_deep_scrub_interval = 2419200
+
+        public_network = ${commaSep cfg.publicNetworks}
+        cluster_network = ${commaSep cfg.clusterNetworks}
+        auth_cluster_required = cephx
+        auth_service_required = cephx
+        auth_client_required = cephx
+
+        # Enforce on-wire transport encryption.
+        ms_cluster_mode = secure
+        ms_service_mode = secure
+        ms_client_mode = secure
+
+        ${cfg.extraConfig}
+      '';
+
+      environment.etc."ceph/${cfg.clusterName}.client.admin.keyring" = {
+        source = cfg.adminKeyring;
+        mode = "0600";
+        # Make ceph own this keyring so that it can use it to get keys for its daemons.
+        user = "ceph";
+        group = "ceph";
+      };
+
+      users.users.ceph = {
+        isNormalUser = false;
+        isSystemUser = true;
+        # TODO: Legacy UID / GID chosen from before we configured the UID declaratively.
+        #       In the future, we whould change this whole module to use
+        #       `config.ids.uids.ceph`, like the upstream nixpkgs Ceph module does.
+        #       Switching away from `nogroup` would also be good as described there.
+        #       For both cases, we'll have to `chown` all relevant existing files on
+        #       deployments, such as `/var/lib/ceph`, and log files.
+        uid = 1001;
+        group = config.users.groups.nogroup.name;
+      };
+      users.groups.ceph = {
+        # TODO: Same TODO as above for the `uid`.
+        gid = 499;
+      };
+
+      # Allow ceph daemons (which run as user ceph) to collect device health metrics.
+      security.sudo.extraRules = map (
+        { sudoersExtraRule, ... }: sudoersExtraRule
+      ) cephMonitoringSudoersCommandsAndPackages;
+
+      # The udevadm trigger/settle in `makeCephOsdSetupSystemdService` waits for these rules rule to be applied.
+      services.udev.extraRules = lib.concatStringsSep "\n" (
+        lib.mapAttrsToList (
+          _localOsdServiceName: osdConfig:
+          ''
+            SUBSYSTEM=="block", ${osdConfig.blockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
+          ''
+          + lib.optionalString (osdConfig.dbBlockDeviceUdevRuleMatcher != null) (''
+            SUBSYSTEM=="block", ${osdConfig.dbBlockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
+          '')
+        ) cfg.osds
+      );
+
+      systemd.services = {
+
+        ceph-mon-setup = mkIf cfg.monitor.enable {
+          description = "Initialize ceph monitor";
+
+          preStart = ensureCephDirs;
+
+          script =
+            let
+              # `--addv` seems currently required to get msgr-v2 working, see:
+              #     https://tracker.ceph.com/issues/53751#note-11
+              monmapNodes = builtins.concatStringsSep " " (
+                lib.concatMap (mon: [
+                  "--addv"
+                  mon.hostname
+                  "[v2:${mon.ipAddress}:3300,v1:${mon.ipAddress}:6789]"
+                ]) cfg.initialMonitors
+              );
+            in
+            # Monitors cannot simply be changed in config, one has to update the monmap, see note [replacing-ceph-monmap-ips-for-existing-cluster]
+            ''
+              set -euo pipefail
+              rm -rf "${monDir}" # Start from scratch.
+              echo "Initializing monitor."
+              MONMAP_DIR=`mktemp -d`
+              ${cfg.package}/bin/monmaptool --create ${monmapNodes} --fsid ${cfg.fsid} "$MONMAP_DIR/monmap"
+              ${cfg.package}/bin/ceph-mon --cluster ${cfg.clusterName} --mkfs -i ${cfg.monitor.nodeName} --monmap "$MONMAP_DIR/monmap" --keyring ${cfg.monitor.initialKeyring}
+              rm -r "$MONMAP_DIR"
+              touch ${monDir}/done
+            '';
+
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            PermissionsStartOnly = true; # only run the script as ceph
+            User = config.users.users.ceph.name;
+            Group = config.users.groups.ceph.name;
+          };
+          unitConfig = {
+            ConditionPathExists = "!${monDir}/done";
+          };
+        };
+
+        ceph-mon = mkIf cfg.monitor.enable {
+          description = "Ceph monitor";
+
+          requires = [ (ensureUnitExists config "ceph-mon-setup.service") ];
+          requiredBy = [ "multi-user.target" ];
+          after = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+            (ensureUnitExists config "ceph-mon-setup.service")
+          ];
+          wants = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+          ];
+
+          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
+
+          path = cephDeviceHealthMonitoringPathsOrPackages;
+
+          preStart = ensureTransientCephDirs;
+
+          serviceConfig = {
+            LimitNOFILE = "1048576";
+            LimitNPROC = "1048576";
+            ExecStart = ''
+              ${cfg.package}/bin/ceph-mon -f --cluster ${cfg.clusterName} --id ${cfg.monitor.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.monitor.bindAddr}" "--public_addr=${cfg.monitor.advertisedPublicAddr}"
+            '';
+            ExecReload = ''
+              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
+            '';
+            PrivateDevices = "yes";
+            ProtectHome = "true";
+            ProtectSystem = "full";
+            PrivateTmp = "true";
+            TasksMax = "infinity";
+            Restart = "on-failure";
+            # StartLimitBurst="5";
+            RestartSec = "10";
+          };
+          # startLimitIntervalSec = 30 * 60;
+        };
+
+        ceph-mgr-setup = mkIf cfg.manager.enable {
+          description = "Initialize Ceph manager";
+
+          preStart = ensureCephDirs;
+
+          script = ''
+            set -euo pipefail
+            mkdir -p ${mgrDir}
+            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
+            do
+              sleep 1
+            done
+            ${cfg.package}/bin/ceph auth get-or-create mgr.${cfg.manager.nodeName} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o ${mgrDir}/keyring
+            touch "${mgrDir}/.nix_done"
+          '';
+
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            PermissionsStartOnly = true; # only run the script as ceph
+            User = config.users.users.ceph.name;
+            Group = config.users.groups.ceph.name;
+          };
+          unitConfig = {
+            ConditionPathExists = "!${mgrDir}/.nix_done";
+          };
+        };
+
+        ceph-mgr = mkIf cfg.manager.enable {
+          description = "Ceph manager";
+
+          requires = [ (ensureUnitExists config "ceph-mgr-setup.service") ];
+          requiredBy = [ "multi-user.target" ];
+          after = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+            (ensureUnitExists config "ceph-mgr-setup.service")
+          ];
+          wants = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+          ];
+
+          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
+
+          preStart = ensureTransientCephDirs;
+
+          serviceConfig = {
+            LimitNOFILE = "1048576";
+            LimitNPROC = "1048576";
+
+            ExecStart = ''
+              ${cfg.package}/bin/ceph-mgr -f --cluster ${cfg.clusterName} --id ${cfg.manager.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name}
+            '';
+            ExecReload = ''
+              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
+            '';
+            Restart = "on-failure";
+            RestartSec = 10;
+            # StartLimitBurst="3";
+          };
+          # startLimitIntervalSec = 30 * 60;
+        };
+
+        ceph-mds-setup = mkIf cfg.mds.enable {
+          description = "Initialize Ceph MDS";
+
+          preStart = ensureCephDirs;
+
+          script = ''
+            set -euo pipefail
+            mkdir -p ${mdsDir}
+            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
+            do
+              sleep 1
+            done
+            ${cfg.package}/bin/ceph auth get-or-create mds.${cfg.mds.nodeName} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o ${mdsDir}/keyring
+            touch "${mdsDir}/.nix_done"
+          '';
+
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            PermissionsStartOnly = true; # only run the script as ceph
+            User = config.users.users.ceph.name;
+            Group = config.users.groups.ceph.name;
+          };
+          unitConfig = {
+            ConditionPathExists = "!${mdsDir}/.nix_done";
+          };
+        };
+
+        ceph-mds = mkIf cfg.mds.enable {
+          description = "Ceph MDS";
+
+          requires = [ (ensureUnitExists config "ceph-mds-setup.service") ];
+          requiredBy = [ "multi-user.target" ];
+          after = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+            (ensureUnitExists config "ceph-mds-setup.service")
+          ];
+          wants = [
+            "network.target"
+            "local-fs.target"
+            "time-sync.target"
+          ];
+
+          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
+
+          preStart = ensureTransientCephDirs;
+
+          serviceConfig = {
+            LimitNOFILE = "1048576";
+            LimitNPROC = "1048576";
+
+            ExecStart = ''
+              ${cfg.package}/bin/ceph-mds -f --cluster ${cfg.clusterName} --id ${cfg.mds.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_addr=${cfg.mds.listenAddr}"
+            '';
+            ExecReload = ''
+              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
+            '';
+            Restart = "on-failure";
+            # StartLimitBurst="3";
+          };
+          # startLimitIntervalSec = 30 * 60;
+        };
+
+      }
+      # Make one OSD service for each configured OSD.
+      // lib.mapAttrs' (
+        localOsdServiceName: osdConfig:
+        nameValuePair "ceph-osd-setup-${localOsdServiceName}" (
+          makeCephOsdSetupSystemdService localOsdServiceName osdConfig
+        )
+      ) cfg.osds
+      // lib.mapAttrs' (
+        localOsdServiceName: osdConfig:
+        nameValuePair "ceph-osd-${localOsdServiceName}" (
+          makeCephOsdSystemdService localOsdServiceName osdConfig
+        )
+      ) cfg.osds;
+    };
+}

modules/nixos/default.nix

@@ -7,6 +7,7 @@
     ./adblock-update.nix
     ./albyhub.nix
     ./backup.nix
+    ./ceph.nix
     ./db.nix
     ./gitea-runner.nix
     ./gnome.nix
@@ -15,7 +16,6 @@
     #./kiwix-serve.nix
     ./kubernetes.nix
     ./linode.nix
-    ./monitoring-access.nix
     ./podman.nix
     ./print.nix
     ./proxy.nix

modules/nixos/gitea-runner.nix

@@ -98,19 +98,6 @@ in
         inherit labels;
         inherit (cfg) name;
         enable = true;
-        hostPackages = with pkgs; [
-          bash
-          buildah
-          coreutils
-          curl
-          gawk
-          gitMinimal
-          gnused
-          nix
-          nodejs
-          podman
-          wget
-        ];
         url = cfg.instanceURL;
         tokenFile = config.age.secrets."gitea-runner-${host}-podman".path;
         settings = {

modules/nixos/home.nix

@@ -1,6 +1,7 @@
 {
   config,
   lib,
+  pkgs,
   ...
 }:
 
@@ -17,9 +18,35 @@ with lib;
   };
 
   config = mkIf cfg {
+    age.secrets.attic.file = ../../secrets/attic.age;
     networking.domain = "thehellings.lan";
     time.timeZone = "America/Chicago";
 
+    systemd.services.attic-client = {
+      enable = true;
+      description = "Attic client watch-store service";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "simple";
+        Restart = "on-failure";
+        RestartSec = "5s";
+      };
+      preStart = ''
+        set -x
+        mkdir -p $XDG_CONFIG_HOME/attic
+        cp ${config.age.secrets.attic.path} $XDG_CONFIG_HOME/attic/config.toml
+      '';
+      script = "${pkgs.attic-client}/bin/attic watch-store --ignore-upstream-cache-filter default";
+      environment = {
+        XDG_CONFIG_HOME = "/var/lib/attic-client";
+      };
+    };
+
+    systemd.tmpfiles.rules = [
+      "d /var/lib/attic-client 0755 root root -"
+    ];
+
     # Open Prometheus exporter ports on LAN-connected hosts only.
     # NOT in baseline.nix to avoid exposing these on internet-facing hosts (e.g. linode).
     networking.firewall.allowedTCPPorts = [

modules/nixos/kubernetes.nix

@@ -112,22 +112,20 @@ in
       keepalived = {
         enable = true;
         openFirewall = true;
-        vrrpInstances = {
-          kubernetes = {
-            interface = cfg.vipInterface;
-            priority = cfg.priority;
-            state = if (config.networking.hostName == "isaiah") then "MASTER" else "BACKUP";
-            virtualIps = [
-              {
-                addr = "${keepaliveIp}/16";
-                dev = cfg.vipInterface;
-              }
-            ];
-            virtualRouterId = 77;
-            extraConfig = ''
-              advert_int 1
-            '';
-          };
+        vrrpInstances.kubernetes = {
+          interface = cfg.vipInterface;
+          priority = cfg.priority;
+          state = if (config.networking.hostName == "isaiah") then "MASTER" else "BACKUP";
+          virtualIps = [
+            {
+              addr = "${keepaliveIp}/16";
+              dev = cfg.vipInterface;
+            }
+          ];
+          virtualRouterId = 77;
+          extraConfig = ''
+            advert_int 1
+          '';
         };
       };
       openiscsi = {

modules/nixos/monitoring-access.nix

@@ -1,53 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-let
-  cfg = config.greg.monitoring-access;
-in
-with lib;
-{
-  options.greg.monitoring-access = {
-    enable = mkOption {
-      type = types.bool;
-      default = true;
-      description = ''
-        Create a dedicated, read-only account (`emily`) for automated
-        monitoring and analysis by the Hermes agent. The account is
-        SSH-key-only (no password set), is not added to `wheel`, and is
-        granted no sudo rights. It only gets read access to the systemd
-        journal via group membership, which is sufficient for log
-        inspection and health/analysis tasks without any privileged
-        access to the rest of the system.
-      '';
-    };
-
-    sshKeys = mkOption {
-      type = types.listOf types.str;
-      default = lib.strings.splitString "\n" (
-        builtins.readFile ../../home/ssh/emily_authorized_keys
-      );
-      description = "SSH public keys authorized to log in as the monitoring account.";
-    };
-  };
-
-  config = mkIf cfg.enable {
-    users.groups.emily = { };
-
-    users.users.emily = {
-      isNormalUser = true;
-      createHome = true;
-      description = "Read-only monitoring/analysis account (Hermes agent)";
-      group = "emily";
-      # No password is set on purpose: this account is SSH-key-only.
-      extraGroups = [
-        "systemd-journal" # read access to the journal for log analysis
-      ];
-      shell = pkgs.bashInteractive;
-      openssh.authorizedKeys.keys = cfg.sshKeys;
-    };
-  };
-}

modules/nixos/vmdev.nix

@@ -23,13 +23,6 @@ with lib;
         type = types.str;
         description = "Kernel module type to install - amd, intel, etc";
       };
-      host = mkOption {
-        type = types.enum [
-          "libvirt"
-          "vbox"
-        ];
-        description = "Which VM hosting type to configure";
-      };
     };
   };
 
@@ -42,6 +35,7 @@ with lib;
       nixos-generators
       packer
       swtpm
+      virt-manager
       virtio-win
       xorriso
     ];
@@ -50,7 +44,7 @@ with lib;
 
     # Enable the virtualisation services
     virtualisation = {
-      libvirtd = mkIf (cfg.host == "libvirt") {
+      libvirtd = {
         enable = true;
         onBoot = "ignore"; # Do not auto-restart VMs on boot, unless they are marked autostart
         qemu = {
@@ -60,11 +54,6 @@ with lib;
           };
         };
       };
-      virtualbox.host = mkIf (cfg.host == "vbox") {
-        enable = true;
-        enableExtensionPack = true;
-        headless = true;
-      };
     };
 
     boot.extraModprobeConfig = "options kvm_${cfg.system} nested=1";

network.json

@@ -9,6 +9,17 @@
     "tailscale": "100.64.0.0/10"
   },
   "hosts": {
+    "builder2": {
+      "ip": "10.42.1.17",
+      "system": "x86_64-linux"
+    },
+    "chronicles": {
+      "ip": "10.42.1.4",
+      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEBecZUva9OnZuXLaBun6/1ITo5f9p0YMLPD+q0egLRS",
+      "external": true,
+      "ts": "100.119.228.115",
+      "aliases": ["s3"]
+    },
     "exodus": {
       "ip": null,
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3",
@@ -24,15 +35,6 @@
       "tags": ["router", "server"],
       "nebulaIp": "10.157.0.2"
     },
-    "gregory.hellings-mbp": {
-      "external": true,
-      "system": "aarch64-darwin",
-      "user": "gregory.hellings"
-    },
-    "gregs-MacBook-Pro-16-inch-Nov-2024": {
-      "external": true,
-      "system": "aarch64-darwin"
-    },
     "hosea": {
       "ip": "10.42.1.7",
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKLIwkTTXA56sUlUjEulXXZRvZy5H4a5ZwgKWLlpkQDz",
@@ -41,6 +43,10 @@
       "tags": ["server"],
       "nebulaIp": "10.157.0.3"
     },
+    "icdm-root": {
+      "external": true,
+      "system": "x86_64-linux"
+    },
     "isaiah": {
       "builder": true,
       "ip": "10.42.1.6",
@@ -55,12 +61,12 @@
       "external": true,
       "system": "x86_64-linux"
     },
-    "ivr": {
+    "gregory.hellings-mbp": {
       "external": true,
-      "system": "aarch64-darwin"
+      "system": "aarch64-darwin",
+      "user": "gregory.hellings"
     },
     "jeremiah": {
-      "aliases": ["buildbot"],
       "builder": true,
       "ip": "10.42.1.8",
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOjQjXq9WYU2Ki27BR9WwJ4ZruS/lJXbjC1b0Q42Adi0",
@@ -70,19 +76,20 @@
       "tags": ["builder", "kube", "server"],
       "nebulaIp": "10.157.0.5"
     },
-    "kuma": {
-      "ip": "10.42.1.19",
-      "nebulaIp": "10.157.0.8",
-      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIhr+LmYMOk4Hixxew2FiAvL8sycgQvnhK8PBGjfnkJb",
-      "system": "x86_64-linux",
-      "tags": ["server"]
+    "joel": {
+      "external": true,
+      "ip": "10.42.0.4",
+      "ts": null
+    },
+    "gregs-MacBook-Pro-16-inch-Nov-2024": {
+      "external": true,
+      "system": "aarch64-darwin"
     },
     "lithic": {
       "external": true,
       "system": "aarch64-darwin"
     },
     "linode": {
-      "connectAddr": "thehellings.com",
       "ip": null,
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMv9Zud3kZOl86gtmkn+uj3D4kiXWDPtyUL02VVLNR4Q",
       "ts": "100.109.86.8",
@@ -90,6 +97,10 @@
       "tags": ["public", "server"],
       "nebulaIp": "10.157.0.1"
     },
+    "ivr": {
+      "external": true,
+      "system": "aarch64-darwin"
+    },
     "MacBook-Pro.local": {
       "external": true,
       "system": "aarch64-darwin"
@@ -98,10 +109,38 @@
       "external": true,
       "system": "aarch64-darwin"
     },
+    "nas1": {
+      "external": true,
+      "ip": "10.42.1.14",
+      "ts": "100.114.187.61"
+    },
     "nixos": {
       "external": true,
       "system": "x86_64-linux"
     },
+    "printer": {
+      "external": true,
+      "ip": "10.42.1.3"
+    },
+    "proxmoxtemplate": {
+      "external": true,
+      "system": "x86_64-linux"
+    },
+    "pve2": {
+      "external": true,
+      "ip": "10.42.1.15",
+      "system": "x86_64-linux"
+    },
+    "pve3": {
+      "external": true,
+      "ip": "10.42.1.16",
+      "system": "x86_64-linux"
+    },
+    "pve4": {
+      "external": true,
+      "ip": "10.42.1.17",
+      "system": "x86_64-linux"
+    },
     "wsl": {
       "external": true,
       "system": "aarch64-linux"
@@ -116,57 +155,5 @@
       "tags": ["builder", "kube", "server"],
       "nebulaIp": "10.157.0.6"
     }
-  },
-  "external": {
-    "chronicles": {
-      "ip": "10.42.1.4",
-      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEBecZUva9OnZuXLaBun6/1ITo5f9p0YMLPD+q0egLRS",
-      "ts": "100.119.228.115",
-      "aliases": ["s3"]
-    },
-    "hermes": {
-      "ip": "10.42.1.18",
-      "mac": "BC:24:11:E4:72:AB",
-      "nebulaIp": "10.157.0.8",
-      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILFYyzz/9i5rXprCQj9IL1ulrbQ6E9BOSeOcvf4D/b0G",
-      "tags": ["server"]
-    },
-    "k3s": {
-      "aliases": ["*.k3s"],
-      "ip": "10.42.5.1",
-      "nebulaIp": "10.157.100.1"
-    },
-    "nas1": {
-      "aliases": ["*.nas1"],
-      "ip": "10.42.1.14",
-      "ts": "100.114.187.61"
-    },
-    "printer": {
-      "ip": "10.42.1.3"
-    },
-    "pve1": {
-      "ip": "10.42.0.4"
-    },
-    "pve2": {
-      "ip": "10.42.1.15"
-    },
-    "pve2bmc": {
-      "ip": "10.42.6.2",
-      "mac": "00:25:90:4b:34:e8"
-    },
-    "pve3": {
-      "ip": "10.42.1.16"
-    },
-    "pve3bmc": {
-      "ip": "10.42.6.3",
-      "mac": "00:25:90:4a:dc:2e"
-    },
-    "pve4": {
-      "ip": "10.42.1.17"
-    },
-    "pve4bmc": {
-      "ip": "10.42.6.4",
-      "mac": "00:25:90:4a:d8:2e"
-    }
   }
 }

pkgs/default.nix

@@ -16,15 +16,12 @@ let
     adblock_update = c ./adblock_update.nix { };
     brew = c ./homebrew.nix { };
     create_ssl = c ./create_ssl.nix { };
-    dockerCompat =
-      pkgs.runCommand "docker-compat"
-        {
-          nativeBuildInputs = [ ];
-        }
-        ''
-          mkdir -p $out/bin
-          ln -s ${pkgs.podman}/bin/podman $out/bin/docker
-        '';
+    dockerCompat = pkgs.runCommand "docker-compat" {
+      nativeBuildInputs  = [];
+    } ''
+      mkdir -p $out/bin
+      ln -s ${pkgs.podman}/bin/podman $out/bin/docker
+    '';
     gcc-tune = c ./gcc-tune.nix { };
     #gen-build = c ./gen-build { };
     hms = c ./hms { };
@@ -32,7 +29,6 @@ let
     inject = c ./inject.nix { };
     setup-ssh = c ./setup-ssh { };
     upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
-    zim-updater = c ./zim/updater.nix { };
   };
   x86Linux = {
     qemu-hook = c ./qemu-hook.nix { };

pkgs/zim/blobs.json

@@ -2,18 +2,18 @@
   "en": {
     "gutenberg": {
       "name": "gutenberg_en_all",
-      "version": "2025-11",
-      "hash": "sha256-AWd8jVVKHKssv9AgrJnryn3Wx0084NE/JmorpgPfryg="
+      "version": "2023-08",
+      "hash": "sha256-OXmdHdsLZcW4nCUQsy7sMpUssS8NV7ztkCgS/nsISuw="
     },
     "phet": {
       "name": "phet_en_all",
-      "version": "2026-05",
-      "hash": "sha256-zAAq/X5rjQUiYmjMpBtWP5z3J2ZHJMmIxFKnxLAhOlA="
+      "version": "2025-03",
+      "hash": "sha256-ARuUzU2o17J2/ZedHc2acXl33dtHCbQJEb72UQUEm1Y="
     },
     "wikibooks": {
       "name": "wikibooks_en_all_maxi",
-      "version": "2026-04",
-      "hash": "sha256-wt7Zr+RkfCsFrOudMCYBADacu6IvPQDkZ6Y0VG2j9hA="
+      "version": "2025-10",
+      "hash": "sha256-ONBh/ze1Fv2Ffm5b9vDzYS/i2cWSa4pM4MLZnozr2n8="
     },
     "wikipedia": {
       "name": "wikipedia_en_all_maxi",
@@ -22,40 +22,40 @@
     },
     "wikisource": {
       "name": "wikisource_en_all_maxi",
-      "version": "2026-05",
-      "hash": "sha256-OA4b+U8mxpcX3fst6hrMyctucYTQOlG/b4qZDiVlcf4="
+      "version": "2025-11",
+      "hash": "sha256-p1Jio+PMTZVSLIDIOBeMG3acXJg83CwQM7zIWDUyozM="
     },
     "wikiversity": {
       "name": "wikiversity_en_all_maxi",
-      "version": "2026-05",
-      "hash": "sha256-8mZ1CSUcF4QnDIyN0M2KedQ4pcSUCmzHBlOm93MYpCE="
+      "version": "2025-11",
+      "hash": "sha256-IG/gAUdc/vHRrIWGHhw8vMJdLWwqrNbfh+SiclQRIAI="
     },
     "wiktionary": {
       "name": "wiktionary_en_all_nopic",
-      "version": "2026-05",
-      "hash": "sha256-Dwiz+viVQt0zb077R9KQRgtUtxG9ixA/DeXAkCdD4rM="
+      "version": "2025-09",
+      "hash": "sha256-Ghcb60qeGaSJtTKQkJoMx/XucX7Lt1XY145lD3gHlMg="
     }
   },
   "fr": {
     "gutenberg": {
       "name": "gutenberg_fr_all",
-      "version": "2026-01",
-      "hash": "sha256-sGn1TeKwBK0+Er5YOJWq6g0itEm4gIGvKxv/PW0DIao="
+      "version": "2025-10",
+      "hash": "sha256-gLHxLXJNwwcxM/mZHtoJ8ojbGQ1fugxgni/+RXccwhU="
     },
     "phet": {
       "name": "phet_fr_all",
-      "version": "2026-05",
-      "hash": "sha256-bJmchFnaPcofE/hy3ZcRggMT9osHqXwLbwMLJ+cpF28="
+      "version": "2025-03",
+      "hash": "sha256-CSboZNTIowLLhp1u9wkxH8xmu3LvQNx5q493AOYJRIc="
     },
     "wikibooks": {
       "name": "wikibooks_fr_all_maxi",
-      "version": "2026-07",
-      "hash": "sha256-csI+E5UFGi42BLGWWHxQhVr5KSihCFc53KGKwo557Ck="
+      "version": "2025-09",
+      "hash": "sha256-KHCc/73L5Bd9iZ47SRV6vpI8fVM1v9hk1TpEvTro2uo="
     },
     "wikipedia": {
       "name": "wikipedia_fr_all_maxi",
-      "version": "2026-05",
-      "hash": "sha256-YUAbGzYYr+c2RQqGjHIF9XL1kNsxDgRKnHK1H7/PtDE="
+      "version": "2025-06",
+      "hash": "sha256-ve7Mbh96/ObNbV/KVNYZpfLN+HdWlJ7C5LItEdVkRH0="
     },
     "wikisource": {
       "name": "wikisource_fr_all_maxi",
@@ -64,25 +64,25 @@
     },
     "wikiversity": {
       "name": "wikiversity_fr_all_maxi",
-      "version": "2026-05",
-      "hash": "sha256-nVDHtkWIOJkoiGixnA1zVR1QS58thjc12FJyagK3Ec0="
+      "version": "2025-09",
+      "hash": "sha256-55fdtw/cRezq6nkRp6wuwkukBiqsQKdwh9QWVAgFcBI="
     },
     "wiktionary": {
       "name": "wiktionary_fr_all_nopic",
-      "version": "2026-05",
-      "hash": "sha256-7UXByW2IIL0hec8RPT39jpg5IEGvRMD47hc4Y4vFkJs="
+      "version": "2025-11",
+      "hash": "sha256-P0CJptee5rDzccxyRDBzyRLuuFOcaLsJYi6DkBFUfIc="
     }
   },
   "ht": {
     "phet": {
       "name": "phet_ht_all",
-      "version": "2026-05",
-      "hash": "sha256-GiJ1jllhioyMYidQ7+Lcbdd9JN2yf04ZQgnO8XaGAqY="
+      "version": "2025-03",
+      "hash": "sha256-bHrPzE8H7ptrP6r5wPlXSsXNlTiKxJ9KatABC4kwx+s="
     },
     "wikipedia": {
       "name": "wikipedia_ht_all_maxi",
-      "version": "2026-07",
-      "hash": "sha256-InS9cMRaIv9ArlKVwKUypz56m4DgSR7Tl6XpSTMzH+o="
+      "version": "2026-04",
+      "hash": "sha256-qUX0pKxIyNsWX4V6kNIsP4Z9nc6TDhwdDR0MmpDOQFs="
     }
   }
 }

pkgs/zim/updater.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"reflect"
+	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"
@@ -17,144 +17,27 @@ import (
 
 const BASE = "https://download.kiwix.org/zim"
 
-// ///////////////////////////////////////////////////////////////////////////
-// ////////////////  THE BLOB ITSELF /////////////////////////////////////////
-// ///////////////////////////////////////////////////////////////////////////
-type Blobs struct {
-	En    Language `json:"en"`
-	Fr    Language `json:"fr"`
-	Ht    Language `json:"ht"`
-	dirty bool
-}
-
-func (b *Blobs) Populate() {
-	done := make(chan bool)
-	waitFor := 0
-	// Launch self-populating efforts
-	blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
-	for f := range blobType.Fields() {
-		if f.IsExported() {
-			//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
-			waitFor += 1
-			go reflect.Indirect(reflect.ValueOf(b)).
-				FieldByName(f.Name).
-				Addr().
-				Interface().
-				(*Language).
-				Populate(strings.ToLower(f.Name), done)
-		}
-	}
-	// Wait until all languages are completed
-	for d := range done {
-		waitFor -= 1
-		if waitFor == 0 {
-			fmt.Println("Completed waiting for all languages")
-			close(done)
-			break
-		}
-		b.dirty = b.dirty || d
+func getTypes() []string {
+	return []string{
+		"phet",
+		"wikipedia",
+		"wiktionary",
+		"wikiversity",
+		"wikisource",
+		"wikibooks",
+		"gutenberg",
+		"ted",
 	}
 }
 
-/////////////////////////////////////////////////////////////////////////////
-//////////////////////// The Language ///////////////////////////////////////
-/////////////////////////////////////////////////////////////////////////////
-
-type Language struct {
-	Gutenberg   *Zim `json:"gutenberg,omitempty"`
-	Phet        *Zim `json:"phet,omitempty"`
-	Ted         *Zim `json:"ted,omitempty"`
-	Wikibooks   *Zim `json:"wikibooks,omitempty"`
-	Wikipedia   *Zim `json:"wikipedia,omitempty"`
-	Wikisource  *Zim `json:"wikisource,omitempty"`
-	Wikiversity *Zim `json:"wikiversity,omitempty"`
-	Wiktionary  *Zim `json:"wiktionary,omitempty"`
-	dirty       bool
-}
-
-func (l *Language) Populate(code string, done chan bool) {
-	childDone := make(chan bool)
-	waitFor := 0
-	languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
-	l.dirty = false
-	for f := range languageType.Fields() {
-		if f.IsExported() {
-			ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
-			    FieldByName(f.Name)
-			if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
-				waitFor += 1
-				go reflect.Indirect(ptrPtrZim).
-				    Addr().
-				    Interface().
-				    (*Zim).
-				    Populate(strings.ToLower(f.Name), code, childDone)
-			} else {
-				// TODO: Figure out how to set a value over the nil pointer
-				// of the field, so we can auto-detect when these are available
-				// in the future
-				// waitFor += 1
-				//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
-				//ptrPtrZim.Set(reflect.ValueOf(z))
-				//go z.Populate(strings.ToLower(f.Name), code, childDone)
-			}
-		}
-	}
-	// Wait until children are all done
-	for d := range childDone {
-		waitFor -= 1
-		if waitFor == 0 {
-			close(childDone)
-			break
-		}
-		l.dirty = l.dirty || d
-	}
-
-	//fmt.Printf("%s\tFinished populate\n", code)
-	done <- l.dirty
-}
-
-// ///////////////////////////////////////////////////////////////////////////
-// ////////////////////// A Single Zim ///////////////////////////////////////
-// ///////////////////////////////////////////////////////////////////////////
-type Zim struct {
-	Name    string `json:"name"`
-	Version string `json:"version"`
-	Hash    string `json:"hash"`
-	dirty   bool
-}
-
-func (z *Zim) Populate(category string, language string, done chan bool) {
-	//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
-	// Look for a possible Zim file
-	links := getLinks(getPage(category))
-	link, err := getName(links, category, language)
-	if err != nil {
-		fmt.Printf("%s:%s\tNo zim found\n", language, category)
-		done <- false
-		return
-	}
-	//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
-	// Extract name and version
-	re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
-	match := re.FindStringSubmatch(link)
-	if len(match) != 3 {
-		fmt.Printf("%s:%s Matches: %s", language, category, match)
-		os.Exit(1)
-	}
-	// Check if the name and version mismatch
-	if match[1] == z.Name && match[2] == z.Version {
-		fmt.Printf("Skipping existing hash: %s\n", link)
-		done <- false
-	} else {
-		z.Name = match[1]
-		z.Version = match[2]
-		// Fetch the Zim file, if it differs from what we currently have
-		z.UpdateHash(category)
-		done <- true
+func getLanguages() []string {
+	return []string{
+		"ht",
+		"en",
+		"fr",
 	}
 }
 
-// Helper function to fetch the HTML of a given type page
 func getPage(t string) string {
 	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
 	if err != nil {
@@ -165,7 +48,6 @@ func getPage(t string) string {
 	return string(pageBytes)
 }
 
-// Helper function to parse out all the links from the page
 func getLinks(page string) []string {
 	ret := []string{}
 
@@ -178,7 +60,6 @@ func getLinks(page string) []string {
 	return ret
 }
 
-// Helper function to get the most likely link for this particular entry
 func getName(links []string, t, lang string) (string, error) {
 	candidates := []string{}
 	prefix := fmt.Sprintf("%s_%s_all", t, lang)
@@ -198,39 +79,44 @@ func getName(links []string, t, lang string) (string, error) {
 	}
 }
 
-// Helper function to get the hash value from the resulting link
-func (z *Zim) UpdateHash(category string) error {
-	file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)
-
-	// First fetch the file into our local Nix store
-	fmt.Printf("Fetching hash: %s\n", file)
-	cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
+func getHash(ch chan result, file, category, language string) {
+	// TODO: Only call this if the file doesn't already have a hash
+	// in the existing file
+	fmt.Printf("Fetching hash for %s\n", file)
+	cmd := exec.Command( "nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
 	out, err := cmd.Output()
 	if err != nil {
-		fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
+		fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
 		if exitErr, ok := err.(*exec.ExitError); ok {
 			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
 		}
-		return errors.New("Error fetching file")
+		ch <- result{category, language, ""}
+		return
 	}
 	hash := strings.TrimSpace(string(out))
-	fmt.Printf("%s: Successfully fetched raw hash\n", file)
-	fmt.Printf("%s: Hash is: %s\n", file, hash)
+	fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
+	ch <- result{category, language, hash}
+}
 
-	// Then, convert the hash to SRI
-	cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
-	out2, err2 := cmd2.Output()
-	if err2 != nil {
-		fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
-		if exitErr, ok := err.(*exec.ExitError); ok {
-			fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
+func outputIsValid(o map[string]map[string]Zim) bool {
+	for a := range o {
+		for b := range o[a] {
+			if o[a][b].Hash == "" {
+				return false
+			}
 		}
-		return errors.New("Error fetching file")
 	}
-	z.Hash = strings.TrimSpace(string(out2))
-	z.dirty = true
-	fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
-	return nil
+	return true
+}
+
+type result struct {
+	category, language, hash string
+}
+
+type Zim struct {
+	Name string `json:"name"`
+	Version string `json:"version"`
+	Hash string `json:"hash"`
 }
 
 func main() {
@@ -239,31 +125,90 @@ func main() {
 
 	// Determine the output file path
 	var outputPath string
-	outputPath = *outputFile
+	if *outputFile != "" {
+		outputPath = *outputFile
+	} else {
+		// Get the directory where updater.go is located
+		execPath, err := os.Executable()
+		if err != nil {
+			// Fallback to current directory if we can't determine executable path
+			outputPath = "blobs.json"
+		} else {
+			dir := filepath.Dir(execPath)
+			outputPath = filepath.Join(dir, "blobs.json")
+		}
+	}
 
 	fmt.Println("Writing file to ", outputPath)
 
 	// Read existing cache if it exists
-	var blobs Blobs
+	cached := make(map[string]map[string]Zim)
 	if data, err := os.ReadFile(outputPath); err == nil {
-		if err := json.Unmarshal(data, &blobs); err != nil {
+		if err := json.Unmarshal(data, &cached); err != nil {
 			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
 		} else {
 			fmt.Printf("Loaded existing cache from %s\n", outputPath)
 		}
-	} else {
-		fmt.Printf("Error reading file: %s", err)
-		os.Exit(1)
 	}
-	blobs.Populate()
 
-	ret, err := json.MarshalIndent(&blobs, "", "  ")
+	output := make(map[string]map[string]Zim)
+	comms := make(chan result)
+	pendingHashes := 0
+
+	for _, t := range getTypes() {
+		page := getPage(t)
+		links := getLinks(page)
+		for _, lang := range getLanguages() {
+			if file, err := getName(links, t, lang); err == nil {
+				if _, ok := output[lang]; !ok {
+					output[lang] = make(map[string]Zim)
+				}
+
+				// Check if this file already exists in cache with same name
+				if cachedLang, ok := cached[lang]; ok {
+					if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
+						// Reuse cached hash
+						fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
+						output[lang][t] = cachedEntry
+						continue
+					}
+				}
+
+				// File is new or name has changed, fetch hash
+				output[lang][t] = Zim{file, ""}
+				pendingHashes++
+				go getHash(comms, file, t, lang)
+			}
+		}
+	}
+
+	// Only wait for results if we actually spawned goroutines
+	if pendingHashes > 0 {
+		hashesReceived := 0
+		for r := range comms {
+			if entry, ok := output[r.language][r.category]; ok {
+				entry.Hash = r.hash
+				output[r.language][r.category] = entry
+			}
+			hashesReceived++
+			if hashesReceived >= pendingHashes {
+				close(comms)
+				break
+			}
+		}
+	}
+
+	// Verify all hashes are present
+	if !outputIsValid(output) {
+		fmt.Println("Warning: Some hashes are missing from the output")
+	}
+	ret, err := json.MarshalIndent(output, "", "  ")
 	if err != nil {
 		fmt.Printf("Error marshaling JSON: %v\n", err)
 		os.Exit(1)
 	}
 
-	err = os.WriteFile(outputPath, []byte(fmt.Sprintf("%s\n", ret)), 0644)
+	err = os.WriteFile(outputPath, ret, 0644)
 	if err != nil {
 		fmt.Printf("Error writing to file %s: %v\n", outputPath, err)
 		os.Exit(1)