Merge pull request 'fix: prevent HAProxy from reusing stale keep-alive conns to nginx/Nextcloud' (#39) from emily/nixos:fix/haproxy-stale-keepalive-nextcloud into main

Reviewed-on: #39

.gitea/workflows/update-zims.yaml

@@ -0,0 +1,51 @@
+name: Update zims pin
+
+"on":
+  schedule:
+    - cron: "0 2 1 * *"  # 0200 on the first of every month
+  workflow_dispatch:
+
+jobs:
+  update-flake-lock:
+    runs-on: nix-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update flake.lock
+        run: nix run .#zim-updater -- --output pkgs/zim/blobs.json
+
+      - name: Create PR if changed
+        env:
+          GITEA_TOKEN: ${{ secrets.KLAATU_TOKEN }}
+          GITEA_URL: https://src.thehellings.com
+          REPO: greg/nixos
+        run: |
+          if git diff --quiet pkgs/zim/blobs.json; then
+            echo "blobs.json unchanged, nothing to do"
+            exit 0
+          fi
+
+          BRANCH="auto/update-zims-$(date +%Y%m%d)"
+          git config user.email "klaatu@thehellings.com"
+          git config user.name "klaatu"
+          git checkout -b "$BRANCH"
+          git add pkgs/zim/blobs.json
+          git commit -m "chore: update zim blobs.json $(date +%Y-%m-%d)"
+
+          # Push branch using token auth
+          git remote set-url origin "https://klaatu:${GITEA_TOKEN}@${GITEA_URL#https://}/${REPO}.git"
+          git push origin "$BRANCH"
+
+          # Create PR via Gitea API
+          curl -s -X POST \
+            -H "Authorization: token ${GITEA_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "${GITEA_URL}/api/v1/repos/${REPO}/pulls" \
+            -d "{
+              \"title\": \"chore: update zims $(date +%Y-%m-%d)\",
+              \"head\": \"$BRANCH\",
+              \"base\": \"main\",
+              \"body\": \"Automated monthly zims update.\\n\\nGenerated by Gitea Actions.\",
+              \"assignees\": [\"greg\"]
+            }"

ca/intermediate_ca.crt

@@ -1,12 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIByDCCAW+gAwIBAgIRANS+dPEH5Vqug7OWhCMmcx4wCgYIKoZIzj0EAwIwLjER
-MA8GA1UEChMISGVsbGluZ3MxGTAXBgNVBAMTEEhlbGxpbmdzIFJvb3QgQ0EwHhcN
-MjQwMjIwMjEyNzIxWhcNMzQwMjE3MjEyNzIxWjA2MREwDwYDVQQKEwhIZWxsaW5n
-czEhMB8GA1UEAxMYSGVsbGluZ3MgSW50ZXJtZWRpYXRlIENBMFkwEwYHKoZIzj0C
-AQYIKoZIzj0DAQcDQgAErZUhPfx5MpNbNVyqHDrIgUGnb6Hitl8hXlpH+kgjBzCi
-7I/+TQnl9Tc0VVNOFOYLOfBi7hV3/QudUtLGk0FKeaNmMGQwDgYDVR0PAQH/BAQD
-AgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFMZR8LDd+hNy+TmtEHvt
-zRzXboh2MB8GA1UdIwQYMBaAFAlH11TwIFGB/K75qZ3e0S+fN3JUMAoGCCqGSM49
-BAMCA0cAMEQCIBxHK6r8pMX5hrTwYKRfMmRIt44m0KtNejA2T5t09hs+AiBfvmHb
-LfoE4qoC6NgpvXorAbx+O7xkem/9svF0Ob+RsA==
------END CERTIFICATE-----

ca/root_ca.crt

@@ -1,11 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIBoTCCAUagAwIBAgIRAL/1mvE8+73nRFGsvzaaVSAwCgYIKoZIzj0EAwIwLjER
-MA8GA1UEChMISGVsbGluZ3MxGTAXBgNVBAMTEEhlbGxpbmdzIFJvb3QgQ0EwHhcN
-MjQwMjIwMjEyNzIwWhcNMzQwMjE3MjEyNzIwWjAuMREwDwYDVQQKEwhIZWxsaW5n
-czEZMBcGA1UEAxMQSGVsbGluZ3MgUm9vdCBDQTBZMBMGByqGSM49AgEGCCqGSM49
-AwEHA0IABEgNBjlT/7gmzNp9vKJvGPJn9/IizuMGVpucdrN9+J1u1ABkLNRzj5p2
-g7s3e/BG+EJnnTf/2tuq3p/wPqmQkdujRTBDMA4GA1UdDwEB/wQEAwIBBjASBgNV
-HRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQWBBQJR9dU8CBRgfyu+amd3tEvnzdyVDAK
-BggqhkjOPQQDAgNJADBGAiEA/uBclcFCOpkEgAeBAVurktYB82sMdIyyDGHcjlwi
-pvkCIQC2kuZ/nXRjibeIebQIVTBskQ1LxlLxnx7zNSjtr3kFfQ==
------END CERTIFICATE-----

compose/attic.yaml

@@ -1,54 +0,0 @@
-services:
-  attic:
-    container_name: attic
-    image: ghcr.io/zhaofengli/attic:latest
-    command: ["-f", "/attic/server.toml"]
-    restart: unless-stopped
-    ports:
-      - 8080:8080
-    networks:
-      attic:
-      pgattic:
-    volumes:
-      - /mnt/all/configs/attic/server.toml:/attic/server.toml
-      - /mnt/all/containers/attic/data:/attic/storage
-    env_file:
-      - stack.env
-    depends_on:
-      pgattic:
-        condition: service_healthy
-    healthcheck:
-      test:
-        [
-          "CMD-SHELL",
-          "wget --no-verbose --tries=1 --spider http://attic:8080 || exit 1",
-        ]
-      interval: 15s
-      timeout: 10s
-      retries: 10
-      start_period: 15s
-    deploy:
-      resources:
-        reservations:
-          cpus: 1.0
-
-  pgattic:
-    container_name: pgattic
-    image: postgres:17.6-alpine
-    restart: unless-stopped
-    ports: []
-    networks:
-      pgattic:
-    volumes:
-      - /mnt/all/containers/attic/postgres:/var/lib/postgresql/data
-    env_file:
-      - stack.env
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-networks:
-  attic:
-  pgattic:

compose/pinchflat.yaml

@@ -1,9 +0,0 @@
-services:
-  pinchflat:
-    image: ghcr.io/kieraneglin/pinchflat:latest
-    ports:
-      - "8945:8945"
-    volumes:
-      - "/mnt/all/configs/pinchflat:/config"
-      - "/mnt/all/video/yt:/downloads"
-    restart: unless-stopped

compose/restic.yaml

@@ -1,19 +0,0 @@
-# Demo of rest-server with prometheus and grafana
-version: "2"
-
-services:
-  restserver:
-    image: "restic/rest-server:0.14.0"
-    volumes:
-      - /mnt/all/backups:/data
-      - /mnt/all/configs/certs:/certs
-    environment:
-      OPTIONS: >-
-        --tls
-        --tls-cert /certs/nas1.shire-zebra.ts.net.crt
-        --tls-key /certs/nas1.shire-zebra.ts.net.key
-        --path /data
-        --prometheus
-        --debug
-    ports:
-      - "30248:8000"

darwin/hosts/MacBook-Prolocal

@@ -1 +1 @@
-gregory
+ivr

darwin/hosts/gregory

@@ -0,0 +1 @@
+ivr

darwin/hosts/gregory.hellings-mbp

@@ -1 +1 @@
-gregory
+ivr

darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024

@@ -0,0 +1 @@
+lithic

darwin/hosts/ivr

@@ -1 +0,0 @@
-gregory/

flake.lock

@@ -31,11 +31,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1780813292,
+        "lastModified": 1783833875,
-        "narHash": "sha256-auBqMf0ROzaSvdO6Ot/vxDzGswZs05sGYOZymi4/aAs=",
+        "narHash": "sha256-G+hRtNJ/Nnr6VFMQp2UZdx/ckJDR/RJoK0Fy/yx1/YY=",
         "owner": "nix-community",
         "repo": "buildbot-nix",
-        "rev": "0c1190ae2c9518d7e7fb631c10a685e2fb83501e",
+        "rev": "147af587241e2af85399402da60a4f44fdb1e5d7",
         "type": "github"
       },
       "original": {
@@ -53,11 +53,11 @@
         "stable": "stable"
       },
       "locked": {
-        "lastModified": 1762034856,
+        "lastModified": 1783909498,
-        "narHash": "sha256-QVey3iP3UEoiFVXgypyjTvCrsIlA4ecx6Acaz5C8/PQ=",
+        "narHash": "sha256-T9OfLPLuh1Bf1xojlpWXwooJ6IXapxvb8GM0p3YNy8g=",
         "owner": "zhaofengli",
         "repo": "colmena",
-        "rev": "349b035a5027f23d88eeb3bc41085d7ee29f18ed",
+        "rev": "76ba0daa542880b730faec81f4e87efcaa63bc57",
         "type": "github"
       },
       "original": {
@@ -95,11 +95,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1780795403,
+        "lastModified": 1784362797,
-        "narHash": "sha256-AkWx4Zt9pQbD/f82Z8N57+d0HGLN/rV3gdMKJTpBPKs=",
+        "narHash": "sha256-EP9b9b+OXDxHBPefFwMYCIaLq0fn3UkmrbfzbLUT7kQ=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "6a771120d607dcccb279a27d227650e324815c35",
+        "rev": "b4cccbd4bc299c1f71ae185b79c3cf99aa82805c",
         "type": "github"
       },
       "original": {
@@ -112,11 +112,11 @@
     "flake-compat": {
       "flake": false,
       "locked": {
-        "lastModified": 1650374568,
+        "lastModified": 1767039857,
-        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
         "type": "github"
       },
       "original": {
@@ -162,11 +162,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1778716662,
+        "lastModified": 1782949081,
-        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -183,11 +183,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1778716662,
+        "lastModified": 1782949081,
-        "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
+        "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
+        "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
         "type": "github"
       },
       "original": {
@@ -218,12 +218,15 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1659877975,
+        "lastModified": 1731533236,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
         "type": "github"
       },
       "original": {
@@ -239,11 +242,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1780885330,
+        "lastModified": 1784407317,
-        "narHash": "sha256-aMA5oAq2Iv467U9s8YOb50DYQT9w0WJbyWqwlzHuLMs=",
+        "narHash": "sha256-iZrxHToDJWnvt+5LGAtvuQMTy1NZYlNEKbKaVtBJNcc=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "4fe95527cbe952713318ada8a4d122e1a6ab120f",
+        "rev": "39411a8e12a5526d992e65bc7e3dc9a4414d6713",
         "type": "github"
       },
       "original": {
@@ -278,14 +281,14 @@
       "inputs": {
         "flake-compat": "flake-compat_2",
         "nixpkgs": "nixpkgs_2",
-        "systems": "systems_2"
+        "systems": "systems_3"
       },
       "locked": {
-        "lastModified": 1780375694,
+        "lastModified": 1784344393,
-        "narHash": "sha256-TznzgYVONg28KiSFB2rVdf/eLVIMtEQOxKt13Kzyrp8=",
+        "narHash": "sha256-yAo2ZzSIdeBZg7cOKUpQxwflL+DRYN+1DMObZk2lP2Q=",
         "owner": "Infinidoge",
         "repo": "nix-minecraft",
-        "rev": "e6f8bec35104ca5955efe73742da58d2823684f7",
+        "rev": "7297d14c52ec8ef39c6aeff2c1818541fd030473",
         "type": "github"
       },
       "original": {
@@ -300,11 +303,11 @@
         "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
-        "lastModified": 1780922506,
+        "lastModified": 1784016977,
-        "narHash": "sha256-/JsYspjkvMXmtcHjZ9o4+rznCuI5RxruJ228TQN5slY=",
+        "narHash": "sha256-TydDba3YD2u15uS0L+PDs7lMqPopnzWggljTKDqOCSw=",
         "owner": "Mic92",
         "repo": "niks3",
-        "rev": "1be2c7feaa5f5d99136e148cb9458971b31b5a88",
+        "rev": "b306808bf381e7e66e33de1e9446a1be0935f4e3",
         "type": "github"
       },
       "original": {
@@ -321,11 +324,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1729742964,
+        "lastModified": 1737420293,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=",
         "owner": "nix-community",
         "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9",
         "type": "github"
       },
       "original": {
@@ -339,11 +342,11 @@
         "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1780310866,
+        "lastModified": 1784310968,
-        "narHash": "sha256-fPBRVf6A5xlACYcOI59shGrjURuvwu0lRsDoSCEXt/I=",
+        "narHash": "sha256-rkSPTePrKqs4dg+i7ZFCq93+HrClac6oSwXX927SVjA=",
         "owner": "nixos",
         "repo": "nixos-hardware",
-        "rev": "4ed851c979641e28597a05086332d75cdc9e395f",
+        "rev": "779c32a00155994c86cde8213a8dd4df139d4355",
         "type": "github"
       },
       "original": {
@@ -354,11 +357,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1750134718,
+        "lastModified": 1783224372,
-        "narHash": "sha256-v263g4GbxXv87hMXMCpjkIxd/viIF7p3JpJrwgKdNiI=",
+        "narHash": "sha256-8i/87eeoqiGE4yOTjwSA3Eh/ziJRQEmd/unYU+K27sk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9e83b64f727c88a7711a2c463a7b16eedb69a84c",
+        "rev": "d407951447dcd00442e97087bf374aad70c04cea",
         "type": "github"
       },
       "original": {
@@ -370,11 +373,11 @@
     },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1777168982,
+        "lastModified": 1782614948,
-        "narHash": "sha256-GOkGPcboWE9BmGCRMLX3worL4EMnsnG8MyKmXNeYuhQ=",
+        "narHash": "sha256-ePjCwr1sNm9NYUqywL7QfK3JnlS015msC+eBu2zKlp8=",
         "owner": "nix-community",
         "repo": "nixpkgs.lib",
-        "rev": "f5901329dade4a6ea039af1433fb087bd9c1fe14",
+        "rev": "db3f255737b94216eb71cce308e2912cf6bc2d7c",
         "type": "github"
       },
       "original": {
@@ -385,11 +388,11 @@
     },
     "nixpkgs-lib_2": {
       "locked": {
-        "lastModified": 1780800655,
+        "lastModified": 1783821755,
-        "narHash": "sha256-eVQuIiDIy24NIAW+yEirKosWHFYAml6dghmevWgruBE=",
+        "narHash": "sha256-eMPX9S6MKPyUnaOgeRfrG7OKUiAlc1AlcRinMbSB0WA=",
         "owner": "nix-community",
         "repo": "nixpkgs.lib",
-        "rev": "78afa8e310f34cba09443a5ef2fb47bfd21d294f",
+        "rev": "228ab8523d81526e57a6ca342e1a919fb6d246a8",
         "type": "github"
       },
       "original": {
@@ -416,11 +419,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1780475387,
+        "lastModified": 1783978241,
-        "narHash": "sha256-JsufhU/MyfEEJMNmtmW0DY6LbOVIjjPS0zGQpHvgrBY=",
+        "narHash": "sha256-7kK0Y/fIV2NTKArkd/eZGaFg+dEmgP8KDRsGK2vB5M4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c8560a2e8ad260841c482fe30731087b92f2223e",
+        "rev": "a8b81d3cc8d35af7bc98694696bea61ad4f8fca7",
         "type": "github"
       },
       "original": {
@@ -445,11 +448,11 @@
     },
     "nixpkgs_5": {
       "locked": {
-        "lastModified": 1780747962,
+        "lastModified": 1783915482,
-        "narHash": "sha256-IX7G1dlKrOqPOImfbo7ADDfV5yU1+j+MRChI3TL4tAA=",
+        "narHash": "sha256-FmieJB8/OUvNxbkboi7+IGfIuSXY3nF/hZQm8kD0r50=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cbb5cf358f50aa6acc9efd6113b7bcfbc352cd73",
+        "rev": "6cdc7fc76e8bf7fde9fa43a849fcaaa70e230dee",
         "type": "github"
       },
       "original": {
@@ -461,11 +464,11 @@
     },
     "nixpkgs_6": {
       "locked": {
-        "lastModified": 1780749050,
+        "lastModified": 1784356753,
-        "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=",
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
         "type": "github"
       },
       "original": {
@@ -493,16 +496,16 @@
     },
     "nixunstable": {
       "locked": {
-        "lastModified": 1780749050,
+        "lastModified": 1784700541,
-        "narHash": "sha256-3av0pIjlOWQ6rDbNOmpUSvbNnJkGORQKKjb4LtCZsIY=",
+        "narHash": "sha256-LcCdjhqwjFVrFTNW6tHm3KNYRrD1TA6bYRea30yIIjw=",
-        "owner": "nixos",
+        "owner": "geri1701",
         "repo": "nixpkgs",
-        "rev": "a799d3e3886da994fa307f817a6bc705ae538eeb",
+        "rev": "3c598184d1f70c5d0beeea8b95d01ab0179e4ef7",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
+        "owner": "geri1701",
-        "ref": "nixos-unstable",
+        "ref": "lego-v5-acme-spike",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -511,14 +514,14 @@
       "inputs": {
         "flake-parts": "flake-parts_2",
         "nixpkgs": "nixpkgs_5",
-        "systems": "systems_3"
+        "systems": "systems_4"
       },
       "locked": {
-        "lastModified": 1780884186,
+        "lastModified": 1784057377,
-        "narHash": "sha256-ueTLp7DofvIZ0BXEWwi8AjP1cCsuEfO445dk6DtmfKc=",
+        "narHash": "sha256-yycNej5//EsRbV10moBoh+/63vXEwZD1ZFEiRm6C9rQ=",
         "owner": "nix-community",
         "repo": "nixvim",
-        "rev": "82f2ec369eb597554a71ec82f33922d01b7dba8f",
+        "rev": "07180a087e4a00720dc0731cbcd8dec796974381",
         "type": "github"
       },
       "original": {
@@ -534,11 +537,11 @@
         "nixpkgs": "nixpkgs_6"
       },
       "locked": {
-        "lastModified": 1780927127,
+        "lastModified": 1784417922,
-        "narHash": "sha256-bnqVaMBjXP9yXyn09Y+ffCh+xl15ukiFZKeR5ikXdZM=",
+        "narHash": "sha256-19XZ56wJXArMKxjY25pKXkNp/FrYHGoo+HuQtW6teSM=",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "537e92f6133a889617f03a84f4d2741c5662f61a",
+        "rev": "2c806d314605495dd7fd75b5950003a062e2b47a",
         "type": "github"
       },
       "original": {
@@ -568,16 +571,16 @@
     },
     "stable": {
       "locked": {
-        "lastModified": 1750133334,
+        "lastModified": 1783625654,
-        "narHash": "sha256-urV51uWH7fVnhIvsZIELIYalMYsyr2FCalvlRTzqWRw=",
+        "narHash": "sha256-pI1244/PJfTyKhlAr2QYQC55vR6UQdnGA0rJUgtO2IQ=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36ab78dab7da2e4e27911007033713bab534187b",
+        "rev": "a0230bd8d5cbd13893b2263918d396a2c7dd0407",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-25.05",
+        "ref": "release-26.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -613,6 +616,21 @@
       }
     },
     "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_4": {
       "locked": {
         "lastModified": 1774449309,
         "narHash": "sha256-brhZ8DmuGtzkCYHJg4HEd602amKm89Y9ytsFZ5uWD1w=",
@@ -675,11 +693,11 @@
         "nixpkgs": "nixpkgs_7"
       },
       "locked": {
-        "lastModified": 1780892930,
+        "lastModified": 1784343266,
-        "narHash": "sha256-YjT3Kf3tEdh6LK7YtW1d3DP02mH+Cf331Xt8CDuBg9s=",
+        "narHash": "sha256-EGkegdTz2n6ESyih8s3dUuPyJQWlYfPp7U41J05g8PY=",
         "owner": "nix-community",
         "repo": "nix-vscode-extensions",
-        "rev": "ebd0101b03e316f3a62089335102d47e4bf65d5a",
+        "rev": "472a3e862c76c64ac3ad75a24d332cb5cdd5f1bb",
         "type": "github"
       },
       "original": {
@@ -696,11 +714,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1780765279,
+        "lastModified": 1784058842,
-        "narHash": "sha256-md6QHmlIx40bQkun43M2eT8aav5GURGkXEMFwof6uZs=",
+        "narHash": "sha256-3u3tvbCIAid3Mv7RrJx13jusIEQC/HeKYhO/SUSxR3A=",
         "owner": "nix-community",
         "repo": "NixOS-WSL",
-        "rev": "3e6d8af994e2a2d31af7a91863d7c0d6e278d951",
+        "rev": "24c8dc8e0f2170e1a377be24dfadc7d9d21dc1ad",
         "type": "github"
       },
       "original": {

flake.nix

@@ -28,7 +28,8 @@
     nix-hardware.url = "github:nixos/nixos-hardware";
     nixpkgs-lib.url = "github:nix-community/nixpkgs.lib";
     nixvimunstable.url = "github:nix-community/nixvim/main";
-    nixunstable.url = "github:nixos/nixpkgs/nixos-unstable";
+    #nixunstable.url = "github:nixos/nixpkgs/nixos-unstable";
+    nixunstable.url = "github:geri1701/nixpkgs/lego-v5-acme-spike";
     nurpkgs.url = "github:nix-community/NUR";
     vsext.url = "github:nix-community/nix-vscode-extensions";
     wsl = {
@@ -100,7 +101,7 @@
                 {
                   deployment = {
                     inherit (v) tags;
-                    targetHost = v.ts;
+                    targetHost = if (v ? "connectAddr") then v.connectAddr else v.nebulaIp;
                     targetUser = "greg";
                   };
                 }

home/baseline/nushell/functions.nu

@@ -12,6 +12,46 @@ def --env unlock [] {
     }
 }
 
+def nebulaIps [] {
+    open /etc/nixos/network.json | get hosts | items { |h, e| $e.nebulaIp? } | where $it != null | sort
+}
+
+def localIps [] {
+    open /etc/nixos/network.json | get hosts | items { |h, e| $e.ip? } | where $it != null | sort
+}
+
+def genNebulaCert [ --ips: string, --name: string ] {
+    let public = $'~/SynologyDrive/nebula/($name).key.pub' | path expand
+    let private = $'~/SynologyDrive/nebula/($name).key' | path expand
+    let cert = $'/etc/nixos/secrets/nebula/($name).crt'
+    let ca_cert = '~/SynologyDrive/nebula/ca.crt' | path expand
+    let ca_key = '~/SynologyDrive/nebula/ca.key' | path expand
+
+    # Generate public key if there isn't one already
+    if ( not ($public | path exists) ) {
+        nebula-cert keygen -out-key $private -out-pub $public
+    }
+    # Clear old cert if there is one
+    if ( $cert | path exists) {
+        rm $cert
+    }
+
+    # Create and sign certs
+    (nebula-cert sign
+        -ca-crt $ca_cert
+        -ca-key $ca_key
+        -name $name
+        -networks $ips
+        -out-crt $cert
+        -in-pub $public
+    )
+
+    # Agenix update
+    cd /etc/nixos/secrets
+    cat $private | agenix -e $'nebula/($name).key.age'
+}
+
+
 def rebuild [ $target: string = "switch" ] {
     if (uname | get operating-system) == "Darwin" {
         sudo darwin-rebuild $target
@@ -35,7 +75,8 @@ def deploy [ $host: string, $build: string = "" ] {
     if $buildhost == "linode" or $buildhost == "genesis" {
         $buildhost = "isaiah"
     }
-    nixos-rebuild switch --sudo --use-substitutes --target-host $host --build-host $buildhost
+    colmena apply --on $host
+    #nixos-rebuild switch --sudo --use-substitutes --target-host $host --build-host $buildhost
 }
 
 def ff [ $file: string ] {

home/baseline/ssh.nix

@@ -27,9 +27,12 @@
 
         "*" = {
           DynamicForward = [ "10240" ];
-          ServerAliveInterval = 60;
+          ForwardAgent = "yes";
           LogLevel = "error";
-          SetEnv = { TERM = "xterm-256color"; };
+          ServerAliveInterval = 60;
+          SetEnv = {
+            TERM = "xterm-256color";
+          };
         };
 
         "10.42.1.4" = lib.hm.dag.entryBefore [ "10.42.*" ] nas;

home/baseline/tools.nix

@@ -8,7 +8,6 @@
   home.packages =
     with pkgs;
     [
-      attic-client
       dig
       jqp
       kubernetes-helm

home/hosts/exodus/default.nix

@@ -23,9 +23,12 @@
     mattermost-desktop
     minio-client
     mumble
+    nebula
     nix-index
+    adoptopenjdk-icedtea-web
     pre-commit
     prismlauncher
+    rclone
     restic
     restic-browser
     tea

home/hosts/icdm-root/default.nix

@@ -1,4 +0,0 @@
-{ ... }:
-
-{
-}

home/ssh/emily_authorized_keys

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBUz4YsVKBERXDT9nl4lwWHoA7NkI7M1Wr3QEYtgz9hy emily-monitoring@thehellings.com

hosts/baseline.nix

@@ -101,6 +101,19 @@
   # Enable the OpenSSH daemon for remote control
   services = {
     locate.enable = true;
+
+    # Defensive rate-limit: cap any single misbehaving service's journal
+    # output fleet-wide. Discovered live on kuma (uptime-kuma logging a
+    # Prometheus-label validation error on every monitor beat, ~100k
+    # lines/hour) that a runaway logger can itself become the obstacle to
+    # incident investigation — journalctl becomes slow/unresponsive and
+    # disk fills — on top of drowning out genuinely useful log signal.
+    # This doesn't fix a specific app's bug, but bounds the blast radius.
+    journald.extraConfig = ''
+      RateLimitIntervalSec=30s
+      RateLimitBurst=2000
+    '';
+
     niks3-auto-upload = {
       enable = config.greg.nix.cache;
       authTokenFile = config.age.secrets.niks3-api-token.path;

hosts/default.nix

@@ -30,6 +30,7 @@ let
       modules = [
         {
           nixpkgs.hostPlatform = system;
+          networking.hostName = name;
         }
         # Imported ones
         top.agenix.nixosModules.default

hosts/unstable/exodus/default.nix

@@ -9,13 +9,9 @@
 {
   imports = [
     ./hardware-configuration.nix
-    top.nix-hardware.nixosModules.framework-11th-gen-intel
+    top.nix-hardware.nixosModules.framework-intel-core-ultra-series1
   ];
 
-  age.secrets = {
-    compose-attic.file = ../../../secrets/compose/attic.env.age;
-  };
-
   boot = {
     loader = {
       systemd-boot = {

hosts/unstable/genesis/net/hosts

@@ -1,12 +1,11 @@
 # Local hosts
 10.42.0.1 switch switch.thehellings.lan  # Core switch for the network
 10.42.0.3 ap ap.thehellings.lan  # OpenWRT access point (static IP)
-10.42.0.4 joel.thehellings.lan  # Proxmox
+10.42.0.4 pve1.thehellings.lan  # Proxmox
 10.42.0.5 sanswitch.thehellings.lan  # Core switch for the SAN
 
 # Home servers
-10.42.1.1 pve1.thehellings.lan
+10.42.1.1 udm router udm.thehellings.lan router.thehellings.lan  # Ubiquiti UDM gateway
-10.42.1.2 opnsense router opnsense.thehellings.lan router.thehellings.lan
 10.42.1.3 printer.thehellings.lan
 10.42.1.4 chronicles chronicles.thehellings.lan nas.thehellings.lan s3.thehellings.lan
 10.42.1.5 genesis genesis.thehellings.lan dns dns.thehellings.lan smart smart.thehellings.lan speedtest.thehellings.lan nixcache.thehellings.lan gitcache.thehellings.lan

hosts/unstable/genesis/networking.nix

@@ -77,7 +77,7 @@ in
       };
     };
     firewall = {
-      enable = false;
+      enable = true;
       allowedUDPPorts = [
         dhcpPort
         dnsPort
@@ -88,7 +88,7 @@ in
         80
       ];
     };
-    nftables.enable = false;
+    nftables.enable = true;
   };
 
   environment.etc."hosts.d/local".text = extraHosts;
@@ -130,10 +130,10 @@ in
         lib.mapAttrs
           (domain: net: {
             master = true;
-            file = makeZoneFile (lib'.hostsByNet net metadata.hosts) domain;
+            file = makeZoneFile (lib'.hostsByNet net (metadata.hosts // metadata.external)) domain;
           })
           {
-            #"shire-zebra.ts.net" = "tailscale";
+            "shire-zebra.ts.net" = "tailscale";
             "nebula.thehellings.com" = "nebula";
             nebula = "nebula";
             "thehellings.lan" = "lan";

hosts/unstable/genesis/networking/dhcp.nix

@@ -60,7 +60,7 @@
         reservations = [
           # Static IPs for personal work
           {
-            hw-address = "00:23:24:72:64:32"; # Joel
+            hw-address = "00:23:24:72:64:32"; # PVE1
             ip-address = "10.42.0.4";
           }
           {
@@ -76,10 +76,6 @@
             #ip-address = "10.42.2.253";
             ip-address = "10.42.100.6";
           }
-          {
-            hw-address = "7c:83:34:b9:ee:ec"; # PVE1
-            ip-address = "10.42.1.1";
-          }
           {
             hw-address = "74:ee:2a:66:b3:51"; # printer
             ip-address = "10.42.1.3";

hosts/unstable/hosea/default.nix

@@ -185,7 +185,7 @@ in
       s3 = {
         accessKeyFile = config.age.secrets.niks3-access-key-id.path;
         bucket = "niks3";
-        endpoint = "nas1.shire-zebra.ts.net:9000";
+        endpoint = "nas1.shire-zebra.ts.net:30188";
         secretKeyFile = config.age.secrets.niks3-secret-access-key.path;
         useSSL = false;
       };

hosts/unstable/icdm-root/boot.nix

@@ -1,11 +0,0 @@
-{ ... }:
-
-{
-  # Bootloader.
-  boot = {
-    loader.grub = {
-      enable = true;
-      device = "/dev/sda";
-    };
-  };
-}

hosts/unstable/icdm-root/default.nix

@@ -1,28 +0,0 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
-{ pkgs, ... }:
-
-{
-  imports = [
-    # Include the results of the hardware scan.
-    ./hardware-configuration.nix
-    ./boot.nix
-    ./filesystem.nix
-    ./location.nix
-    ./networking.nix
-    ./wiki.nix
-  ];
-
-  # Define a user account. Don't forget to set a password with ‘passwd’.
-  users.users.greg = {
-    isNormalUser = true;
-    description = "Gregory Hellings";
-    extraGroups = [
-      "networkmanager"
-      "wheel"
-    ];
-    packages = with pkgs; [ ];
-  };
-}

hosts/unstable/icdm-root/filesystem.nix

@@ -1,13 +0,0 @@
-{ ... }:
-
-let
-in
-{
-  fileSystems."serve" = {
-    #device = "10.42.1.4:/volume1/icdm-mysql/";
-    #fsType = "nfs";
-    device = "/dev/sdb1";
-    fsType = "auto";
-    mountPoint = "/srv";
-  };
-}

hosts/unstable/icdm-root/hardware-configuration.nix

@@ -1,53 +0,0 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{
-  config,
-  lib,
-  modulesPath,
-  ...
-}:
-
-{
-  imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];
-
-  boot.initrd.availableKernelModules = [
-    "xhci_pci"
-    "ehci_pci"
-    "ahci"
-    "usbhid"
-    "usb_storage"
-    "sd_mod"
-  ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-intel" ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" = {
-    device = "/dev/disk/by-uuid/dab0d455-e25e-4445-8fa4-5320047d7e7b";
-    fsType = "btrfs";
-    options = [ "subvol=@" ];
-  };
-
-  fileSystems."/boot" = {
-    device = "/dev/disk/by-uuid/5aedbb07-5761-423b-909d-2560405eae32";
-    fsType = "ext4";
-  };
-
-  fileSystems."/var" = {
-    device = "/dev/disk/by-uuid/57968536-c29d-417d-997e-85223d1d1f65";
-    fsType = "btrfs";
-  };
-
-  swapDevices = [ { device = "/dev/disk/by-uuid/09691dce-375a-43c6-8d40-4498d20a6d9a"; } ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
-  # networking.interfaces.wlp2s0.useDHCP = lib.mkDefault true;
-
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
-}

hosts/unstable/icdm-root/location.nix

@@ -1,15 +0,0 @@
-{ ... }:
-
-{
-  # Set your time zone.
-  time.timeZone = "America/Chicago";
-
-  # Select internationalisation properties.
-  i18n.defaultLocale = "en_US.UTF-8";
-
-  # Configure keymap in X11
-  services.xserver.xkb = {
-    layout = "us";
-    variant = "";
-  };
-}

hosts/unstable/icdm-root/networking.nix

@@ -1,63 +0,0 @@
-{ ... }:
-let
-  dnsHosts = builtins.concatStringsSep "\n" [ "wiki.icdm.lan 10.42.101.1" ];
-in
-{
-  # If we have to do proxying in Bayonnais, we can start to work on that here
-  # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain";
-  networking = {
-    hostName = "icdm-root";
-    useDHCP = false;
-    defaultGateway = "10.42.1.1";
-    nameservers = [
-      "100.100.100.100"
-      "10.42.1.2"
-    ];
-    enableIPv6 = false;
-
-    interfaces = {
-      eno1.ipv4.addresses = [
-        {
-          address = "10.42.101.1";
-          prefixLength = 16;
-        }
-        {
-          address = "10.77.1.2";
-          prefixLength = 16;
-        }
-      ];
-    };
-    # Allow traffic through
-    firewall = {
-      enable = true;
-      allowedTCPPorts = [ 53 ];
-      allowedUDPPorts = [
-        53
-        67
-      ];
-    };
-
-    extraHosts = "${dnsHosts}";
-  };
-
-  services.dnsmasq = {
-    enable = true;
-    settings = {
-      domain = "icdm.lan";
-      dhcp-range = [ "eno1,10.77.1.10,10.77.1.255,255.255.0.0,12h" ];
-      dhcp-option = [
-        "eno1,option:router,10.77.1.1"
-        "eno1,option:dns-server,10.77.1.2,1.1.1.1"
-        "eno1,option:domain-search,icdm.lan"
-      ];
-      expand-hosts = true;
-      log-dhcp = true;
-      log-queries = true;
-      # Upstream servers
-      server = [
-        "1.1.1.1"
-        "8.8.4.4"
-      ];
-    };
-  };
-}

hosts/unstable/icdm-root/wiki.nix

@@ -1,17 +0,0 @@
-{ pkgs, ... }:
-let
-  wikiHost = "wiki.icdm.lan";
-  kiwixport = 8080;
-in
-{
-  services.kiwix-serve = {
-    enable = true;
-    port = kiwixport;
-    library = {
-      inherit (pkgs) zim;
-    };
-  };
-
-  greg.proxies."${wikiHost}".target = "http://localhost:${toString kiwixport}";
-  networking.firewall.allowedTCPPorts = [ 80 ];
-}

hosts/unstable/isaiah/default.nix

@@ -47,6 +47,10 @@
       enable = true;
       extraLabels = [ "bare-metal:host" ];
     };
+    vmdev = {
+      enable = true;
+      host = "libvirt";
+    };
   };
 
   networking = {

hosts/unstable/jeremiah/default.nix

@@ -88,6 +88,7 @@ in
       priority = 254;
     };
     nebula.enable = true;
+    proxies."buildbot.nebula.thehellings.com".target = "http://buildbot.nebula.thehellings.com:8010/";
     tailscale = {
       enable = true;
       tags = [ "home" ];
@@ -142,12 +143,12 @@ in
             updateOutputs = false;
           };
         };
-        domain = "${config.networking.hostName}.shire-zebra.ts.net:8010";
+        domain = "buildbot.nebula.thehellings.com:8010";
         evalMaxMemorySize = 8192;
         evalWorkerCount = 4;
         gitea = {
           enable = true;
-          instanceUrl = "https://gitea.shire-zebra.ts.net";
+          instanceUrl = "https://src.thehellings.com";
           oauthId = "7ec9107d-379b-47c8-870f-1191956d0500";
           oauthSecretFile = config.age.secrets.gitea-oauthSecret.path;
           tokenFile = config.age.secrets.gitea-oauthToken.path;
@@ -155,7 +156,7 @@ in
           webhookSecretFile = config.age.secrets.gitea-webhookSecret.path;
         };
         showTrace = true;
-        #webhookBaseUrl = "http://${config.networking.hostName}.shire-zebra.ts.net:8010";
+        #webhookBaseUrl = "http://${config.networking.hostName}.nebula.thehellings.com:8010";
         workersFile = config.age.secrets.gitea-buildbotWorkersFile.path;
       };
       worker = {

hosts/unstable/kuma/default.nix

@@ -0,0 +1,76 @@
+{
+  config,
+  metadata,
+  modulesPath,
+  pkgs,
+  ...
+}:
+{
+  imports = [ "${modulesPath}/virtualisation/proxmox-image.nix" ];
+  greg = {
+    home = true;
+    nebula.enable = true;
+    proxies =
+      let
+        tgt = {
+          target = "http://localhost:${config.services.uptime-kuma.settings.PORT}";
+          genAliases = false;
+        };
+      in
+      {
+        "kuma.nebula.thehellings.com" = tgt;
+        "kuma.thehellings.lan" = tgt;
+        "kuma.shire-zebra.ts.net" = tgt;
+      };
+  };
+  nix.settings = {
+    sandbox = false;
+  };
+  networking = {
+    defaultGateway = metadata.infra.gw;
+    nameservers = [ metadata.infra.dns ];
+    interfaces.ens18 = {
+      useDHCP = false;
+      ipv4.addresses = [
+        {
+          address = metadata.hosts."${config.networking.hostName}".ip;
+          prefixLength = 16;
+        }
+      ];
+    };
+  };
+  proxmox.cloudInit.enable = false;
+  services = {
+    fstrim.enable = true;
+    mysql = {
+      enable = true;
+      ensureDatabases = [
+        config.services.uptime-kuma.settings.UPTIME_KUMA_DB_NAME
+      ];
+      ensureUsers = [
+        {
+          name = config.services.uptime-kuma.settings.UPTIME_KUMA_DB_USERNAME;
+          ensurePermissions = {
+            "uptimekuma.*" = "ALL PRIVILEGES";
+          };
+        }
+      ];
+      package = pkgs.mariadb;
+    };
+    openssh = {
+      enable = true;
+      openFirewall = true;
+    };
+    uptime-kuma = {
+      enable = true;
+      settings = {
+        PORT = "3001"; # Default, but this allows us to explicitly use it elsewhere
+        UPTIME_KUMA_DB_TYPE = "mariadb";
+        UPTIME_KUMA_DB_SOCKET = "/run/mysqld/mysqld.sock";
+        UPTIME_KUMA_DB_NAME = "uptimekuma";
+        UPTIME_KUMA_DB_USERNAME = "uptimekuma";
+        UPTIME_KUMA_DB_PASSWORD = "uptimekuma";
+      };
+    };
+  };
+}

hosts/unstable/linode/default.nix

@@ -1,34 +1,93 @@
 {
-  pkgs,
-  lib,
   config,
+  lib,
+  metadata,
+  pkgs,
+  pkgs',
   ...
 }:
 
+let
+  homepage = "127.0.0.1:30080";
+  nextcloudPort = 8080;
+  sshPort = 2222;
+  matrixServer = pkgs.writeText "matrix_server" (
+    builtins.toJSON {
+      "m.server" = "matrix.thehellings.com:443";
+    }
+  );
+  matrixClient = pkgs.writeText "matrix_client" (
+    builtins.toJSON {
+      "m.homeserver" = {
+        base_url = "https://matrix.thehellings.com";
+      };
+      "m.identity_server" = {
+        base_url = "https://vector.im";
+      };
+    }
+  );
+in
 {
   imports = [
-    ./git.nix
     ./hardware-configuration.nix
-    ./podman.nix
-    ./matrix.nix
-    ./nextcloud.nix
-    ./nginx.nix
-    ./postgres.nix
   ];
 
+  age.secrets = {
+    acme.file = ../../../secrets/acme.age;
+    nextcloudadmin = {
+      file = ../../../secrets/nextcloudadmin.age;
+      owner = "nextcloud";
+    };
+  };
+
   environment.systemPackages = with pkgs; [
     bind
     graphviz
     nix-du
     pgloader
+    podman-compose
+    pkgs'.upgrade-pg-cluster
   ];
 
+  # Historical per-interface bandwidth tracking (5-min granularity, kept for
+  # months). This is what's actually missing when diagnosing "traffic was
+  # high for the past several hours" reports after the fact — journalctl
+  # timestamps only tell you what else was happening, not the traffic curve
+  # itself. `vnstat -h`/`vnstat --json h` gives an immediate confirm/deny of
+  # a reported window without waiting on live sampling.
+  services.vnstat.enable = true;
+
   greg = {
+    backup.jobs = {
+      nextcloud-bkup = {
+        src = "/var/lib/nextcloud";
+        dest = "nextcloud-backup";
+        pre = lib.getExe (
+          pkgs.writeShellApplication {
+            name = "nextcloud-backup-pre";
+            runtimeInputs = [ config.services.nextcloud.occ ];
+            text = "nextcloud-occ maintenance:mode --on";
+          }
+        );
+        post = lib.getExe (
+          pkgs.writeShellApplication {
+            name = "nextcloud-backup-post";
+            runtimeInputs = [ config.services.nextcloud.occ ];
+            text = "nextcloud-occ maintenance:mode --off";
+          }
+        );
+      };
+      greg-postgresql-backup = {
+        src = config.services.postgresqlBackup.location;
+        dest = "linode-postgres";
+      };
+    };
     gitea-runner = {
       enable = true;
-      extraLabels = [
+      labels = [
         "vps:host"
         "blog:host"
+        "nixos-linode:host"
       ];
     };
     home = false;
@@ -36,20 +95,29 @@
     nebula = {
       enable = true;
       isLighthouse = true;
-    };
+      unsafeRoutes = [
-    proxies."immich.thehellings.com" = {
+        {
-      genAliases = false;
+          route = "10.42.0.0/16";
-      target = "http://localhost:${builtins.toString config.services.immich-public-proxy.port}";
+          via = metadata.hosts.genesis.nebulaIp;
-      ssl = true;
+        }
+      ];
     };
     tailscale.enable = true;
   };
 
   networking = {
-    networkmanager.enable = lib.mkForce false;
-    hostName = "linode";
     domain = "thehellings.com";
-    nameservers = [ "100.88.91.27" ];
+    firewall.allowedTCPPorts = [
+      sshPort
+      80
+      443
+    ];
+    hostName = "linode";
+    nameservers = [
+      "10.157.0.2"
+      "100.96.198.104"
+    ];
+    networkmanager.enable = lib.mkForce false;
   };
 
   programs.ssh.extraConfig = lib.strings.concatStringsSep "\n" [
@@ -60,10 +128,297 @@
     "    UserKnownHostsFile /dev/null"
   ];
 
+  security.acme = {
+    acceptTerms = true;
+    defaults = {
+      dnsPropagationCheck = false;
+      dnsResolver = "92.123.95.3:53,92.123.94.3:53,92.123.94.2:53,92.123.95.4:53,92.123.95.2:53";
+      email = "greg.hellings@gmail.com";
+      extraLegoRunFlags = [ "--ipv4only" ]; # Force IPv4 only
+      #server = "https://acme-staging-v02.api.letsencrypt.org/directory";
+    };
+    certs."thehellings.com" = {
+      dnsProvider = "linode";
+      environmentFile = config.age.secrets.acme.path;
+      extraDomainNames = [
+        "*.thehellings.com"
+      ];
+    };
+  };
+
   services = {
+
+    anubis = {
+      instances = {
+        git = {
+          enable = true;
+          settings = {
+            BIND = "/run/anubis/anubis-git/anubis.sock";
+            COOKIE_DOMAIN = "thehellings.com";
+            SERVE_ROBOTS_TXT = true;
+            SLOG_LEVEL = "DEBUG";
+            TARGET = "http://git.k3s.thehellings.lan";
+          };
+        };
+      };
+    };
+
+    haproxy = {
+      enable = true;
+      config = ''
+        global
+          nbthread 4
+          maxconn 80
+          log /dev/log local0
+
+        defaults
+          log global
+          timeout connect 500s
+          timeout client 500s
+          timeout server 1h
+          # HAProxy defaults to end-to-end keep-alive (client AND server side)
+          # unless a proxy overrides it. Bound how long an idle client-facing
+          # keep-alive connection is held: maxconn is only 80, and leaving
+          # this unset falls back to "timeout client" (500s), which is far
+          # longer than needed just to wait for a pipelined next request.
+          timeout http-keep-alive 30s
+
+        listen gitsshd
+          bind *:${toString sshPort}
+          timeout client 1h
+          mode tcp
+          server git-isaiah isaiah.thehellings.lan:32222
+          server git-jeremiah jeremiah.thehellings.lan:32222
+          server git-zeke zeke.thehellings.lan:32222
+
+        listen stats
+          bind 127.0.0.1:8404
+          stats enable
+          stats uri /
+          stats refresh 10s
+
+        frontend https
+          bind *:80
+          bind *:443 ssl crt ${config.security.acme.certs."thehellings.com".directory}/full.pem
+
+          http-request redirect scheme https unless { ssl_fc }
+          http-request add-header X-Forwarded-Proto https
+
+          http-response replace-header ^Set-Cookie:\ (.*) Set-Cookie \1;\ Secure
+
+          option http-server-close
+          option http-keep-alive
+          option httplog
+
+          #declare capture response len 80
+          #http-response capture res.hdr(Location) id 0
+
+          use_backend git if { hdr(host) -i src.thehellings.com }
+          use_backend git if { req_ssl_sni -i src.thehellings.com }
+          use_backend next if { hdr(host) -i next.thehellings.com }
+          use_backend next if { req_ssl_sni -i next.thehellings.com }
+          use_backend matrix if { hdr(host) -i matrix.thehellings.com }
+          use_backend matrix if { req_ssl_sni -i matrix.thehellings.com }
+          use_backend immich if { hdr(host) -i immich.thehellings.com }
+          use_backend immich if { req_ssl_sni -i immich.thehellings.com }
+          use_backend web if { hdr(host) -i thehellings.com }
+          use_backend web if { req_ssl_sni -i thehellings.com }
+
+        backend git
+          mode http
+          balance roundrobin
+          option accept-unsafe-violations-in-http-response
+          retries 3
+          option forwardfor
+          http-request set-header Host git.k3s.thehellings.lan
+          server git-isaiah isaiah.thehellings.lan:80
+          server git-jeremiah jeremiah.thehellings.lan:80
+          server git-zeke zeke.thehellings.lan:80
+
+        backend immich
+          mode http
+          balance roundrobin
+          option accept-unsafe-violations-in-http-response
+          retries 3
+          option forwardfor
+          server immich-proxy 127.0.0.1:${builtins.toString config.services.immich-public-proxy.port}
+
+        backend matrix
+          mode http
+          balance roundrobin
+          option accept-unsafe-violations-in-http-response
+          retries 3
+          option forwardfor
+          http-request set-header Host matrix.k3s.thehellings.lan
+          server git-isaiah isaiah.thehellings.lan:80
+          server git-jeremiah jeremiah.thehellings.lan:80
+          server git-zeke zeke.thehellings.lan:80
+
+        backend web
+          mode http
+          balance roundrobin
+          option accept-unsafe-violations-in-http-response
+          retries 3
+          option forwardfor
+          http-request return status 200 content-type "application/json" file ${matrixClient} hdr "cache-control" "no-cache" if { path /.well-known/matrix/client }
+          http-request return status 200 content-type "application/json" file ${matrixServer} hdr "cache-control" "no-cache" if { path /.well-known/matrix/server }
+          server web-container ${homepage}
+
+        backend next
+          log global
+          mode http
+          balance roundrobin
+          option accept-unsafe-violations-in-http-response
+          retries 3
+          option forwardfor
+          # nginx (the actual listener on 127.0.0.1:8080) has
+          # keepalive_timeout 65s and will silently close an idle backend
+          # socket after that. HAProxy's default mode is end-to-end
+          # keep-alive, so without this it will happily try to reuse a
+          # backend connection nginx already closed once a mobile client's
+          # own (longer) keep-alive idle assumption outlives 65s - producing
+          # exactly the "unexpected end of stream" / EOFException the
+          # CalDAV/CardDAV client saw. Since the backend is localhost, the
+          # cost of a fresh TCP connection per request is negligible, so
+          # just don't try to reuse them here.
+          option http-server-close
+          #http-response replace-value Location http://localhost:${builtins.toString nextcloudPort}/(.*) https://next.thehellings.com/\2
+          server nextcloud 127.0.0.1:${builtins.toString nextcloudPort}
+      '';
+    };
+
     immich-public-proxy = {
       enable = true;
-      immichUrl = "https://immich.shire-zebra.ts.net";
+      immichUrl = "http://immich.k3s.thehellings.lan";
     };
+
+    logrotate = {
+      enable = true;
+      settings = {
+        postgresBackup = {
+          enable = true;
+          files = "${config.services.postgresqlBackup.location}/*.gz";
+        };
+        postgresLog = {
+          enable = true;
+          files = "/var/lib/postgresql/*/log/*.log";
+          compress = true;
+          compresscmd = "${pkgs.xz}/bin/xz";
+        };
+      };
+    };
+
+    nextcloud = {
+      enable = true;
+      package = pkgs.nextcloud33;
+      appstoreEnable = true;
+      hostName = "127.0.0.1";
+      https = false;
+      config = {
+        adminpassFile = config.age.secrets.nextcloudadmin.path;
+        adminuser = "greg";
+        dbhost = "/run/postgresql";
+        dbtype = "pgsql";
+      };
+      settings = {
+        default_phone_region = "US";
+        overwriteprotocol = "http";
+        trusted_domains = [ "next.thehellings.com" ];
+        trusted_proxies = [
+          "localhost"
+          "127.0.0.1"
+        ];
+      };
+    };
+
+    # Move to :8080 so that we can run haproxy as the primary HTTP service
+    nginx = {
+      virtualHosts."${config.services.nextcloud.hostName}".listen = [
+        {
+          addr = "127.0.0.1";
+          port = nextcloudPort;
+        }
+      ];
+
+      # Route nginx access logs through syslog/journald (rather than only to
+      # /var/log/nginx/access.log, which the read-only monitoring account
+      # can't read) so `journalctl -t nginx_access` gives visibility into
+      # Nextcloud request traffic during bandwidth investigations.
+      #
+      # NOTE: nginx's syslog "tag" only allows alphanumeric characters and
+      # underscores (no hyphens) - an earlier version of this used
+      # tag=nginx-access, which fails nginx's config test with:
+      #   nginx: [emerg] syslog "tag" only allows alphanumeric characters
+      #   and underscore in .../nginx.conf:114
+      # That broke nginx.service (and, transitively, Nextcloud/next.thehellings.com,
+      # which is proxied through nginx on 127.0.0.1:8080) until nginx hit its
+      # systemd restart limit and gave up (start-limit-hit).
+      appendHttpConfig = ''
+        access_log syslog:server=unix:/dev/log,tag=nginx_access combined;
+      '';
+    };
+
+    openssh.settings.PasswordAuthentication = false;
+
+    postgresql = {
+      enable = true;
+      package = pkgs.postgresql_15;
+      checkConfig = true;
+      ensureDatabases = [ "nextcloud" ];
+      #initialScript = pkgs.writeText "create-matrix-db.sql" ''
+      #	CREATE ROLE "matrix-synapse" WITH LOGIN;
+      #	CREATE DATABASE "synapse" WITH OWNER "matrix-synapse" TEMPLATE template0 LC_COLLATE = "C" LC_CTYPE = "C";
+      #	GRANT ALL PRIVILEGES ON DATABASE "synapse" TO "matrix-synapse";
+      #'';  # These are done manually in order to set the LC_COLLATE values properly
+      ensureUsers = [
+        {
+          name = "nextcloud";
+          ensureDBOwnership = true;
+        }
+      ];
+      settings = {
+        log_connections = true;
+        log_statement = "all";
+        logging_collector = true;
+        log_filename = "postgresql.log";
+      };
+      identMap = ''
+        root root postgres
+      '';
+    };
+
+    postgresqlBackup = {
+      enable = true;
+      databases = [ "nextcloud" ];
+    };
+  };
+
+  systemd.services = {
+    haproxy = {
+      after = [
+        "nextcloud.service"
+        "network-online.target"
+      ];
+      wants = [
+        "nextcloud.service"
+        "network-online.target"
+      ];
+    };
+  };
+
+  users.users.haproxy.extraGroups = [ config.security.acme.certs."thehellings.com".group ];
+
+  # Actually serve the content from here
+  virtualisation.oci-containers = {
+    backend = "podman";
+    containers."homepage" = {
+      image = "src.thehellings.com/greg/homepage:latest";
+      ports = [ "${homepage}:80" ];
+    };
+  };
+  virtualisation.podman = {
+    enable = true;
+    dockerCompat = true;
+    dockerSocket.enable = true;
   };
 }

hosts/unstable/linode/git.nix

@@ -1,148 +0,0 @@
-{ config, ... }:
-
-let
-  srcDomain = "src.thehellings.com";
-  sshPort = 2222;
-in
-{
-  greg.proxies."${srcDomain}" = {
-    target = "http://unix:${config.services.anubis.instances.git.settings.BIND}";
-    ssl = true;
-    genAliases = false;
-    extraConfig = ''
-      #proxy_ssl_verify off;
-      #proxy_ssl_server_name on;
-
-      proxy_set_header X-Real-IP $remote_addr;
-      proxy_set_header X-Http-Version $server_protocol;
-      proxy_set_header User-Agent $http_user_agent;
-      client_max_body_size 100000m;
-
-      #proxy_set_header Host $host;
-      #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      #proxy_set_header X-Forwarded-Proto $scheme;
-      #proxy_set_header X-Forwarded-Ssl on;
-
-      # Ultimate AI Block List v1.7 20250924
-      # https://perishablepress.com/ultimate-ai-block-list/
-
-      if ($http_user_agent ~* "(openai\.com|\.ai|-ai|_ai|ai\.|ai-|ai_|ai=|AddSearchBot|Agentic|AgentQL|Agent\ 3|Agent\ API|AI\ Agent|AI\ Article\ Writer|AI\ Chat|AI\ Content\ Detector|AI\ Detection|AI\ Dungeon|AI\ Journalist|AI\ Legion)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(AI\ RAG|AI\ Search|AI\ SEO\ Crawler|AI\ Training|AI\ Web|AI\ Writer|AI2|AIBot|aiHitBot|AIMatrix|AISearch|AITraining|Alexa|Alice\ Yandex|AliGenie|AliyunSec|Alpha\ AI|AlphaAI|Amazon|Amelia)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(AndersPinkBot|AndiBot|Anonymous\ AI|Anthropic|AnyPicker|Anyword|Applebot|Aria\ AI|Aria\ Browse|Articoolo|Ask\ AI|AutoGen|AutoGLM|Automated\ Writer|AutoML|Autonomous\ RAG|AwarioRssBot|AwarioSmartBot|AWS\ Trainium|Azure)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(BabyAGI|BabyCatAGI|BardBot|Basic\ RAG|Bedrock|Big\ Sur|Bigsur|Botsonic|Brightbot|Browser\ MCP\ Agent|Browser\ Use|Bytebot|ByteDance|Bytespider|CarynAI|CatBoost|CC-Crawler|CCBot|Chai|Character)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Charstar\ AI|Chatbot|ChatGLM|Chatsonic|ChatUser|Chinchilla|Claude|ClearScope|Clearview|Cognitive\ AI|Cohere|Common\ Crawl|CommonCrawl|Content\ Harmony|Content\ King|Content\ Optimizer|Content\ Samurai|ContentAtScale|ContentBot|Contentedge)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(ContentShake|Conversion\ AI|Copilot|CopyAI|Copymatic|Copyscape|CoreWeave|Corrective\ RAG|Cotoyogi|CRAB|Crawl4AI|CrawlQ\ AI|Crawlspace|Crew\ AI|CrewAI|Crushon\ AI|DALL-E|DarkBard|DataFor|DataProvider)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Datenbank\ Crawler|DeepAI|Deep\ AI|DeepL|DeepMind|Deep\ Research|DeepResearch|DeepSeek|Devin|Diffbot|Doubao\ AI|DuckAssistBot|DuckDuckGo\ Chat|DuckDuckGo-Enhanced|Echobot|Echobox|Elixir|FacebookBot|FacebookExternalHit|Factset)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Falcon|FIRE-1|Firebase|Firecrawl|Flux|Flyriver|Frase\ AI|FriendlyCrawler|Gato|Gemini|Gemma|Gen\ AI|GenAI|Generative|Genspark|Gentoo-chat|Ghostwriter|GigaChat|GLM|GodMode)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Goose|GPT|Grammarly|Grendizer|Grok|GT\ Bot|GTBot|GTP|Hemingway\ Editor|Hetzner|Hugging|Hunyuan|Hybrid\ Search\ RAG|Hypotenuse\ AI|iAsk|ICC-Crawler|ImageGen|ImagesiftBot|img2dataset|imgproxy)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(INK\ Editor|INKforall|Instructor|IntelliSeek|Inferkit|ISSCyberRiskCrawler|Janitor\ AI|Jasper|Jenni\ AI|Julius\ AI|Kafkai|Kaggle|Kangaroo|Keyword\ Density\ AI|Kimi|Knowledge|KomoBot|Kruti|LangChain|Le\ Chat)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Lensa|Lightpanda|LinerBot|LLaMA|LLM|Local\ RAG\ Agent|Lovable|Magistral|magpie-crawler|Manus|MarketMuse|Meltwater|Meta-AI|Meta-External|Meta-Webindexer|Meta\ AI|MetaAI|MetaTagBot|Middleware|Midjourney)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Mini\ AGI|MiniMax|Mintlify|Mistral|Mixtral|model-training|Monica|Narrative|NeevaBot|netEstate|Neural\ Text|NeuralSEO|NinjaAI|NodeZero|Nova\ Act|NovaAct|OAI-SearchBot|OAI\ SearchBot|OASIS|Olivia)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Omgili|Open\ AI|Open\ Interpreter|OpenAGI|OpenAI|OpenBot|OpenPi|OpenRouter|OpenText\ AI|Operator|Outwrite|Page\ Analyzer\ AI|PanguBot|Panscient|Paperlibot|Paraphraser\.io|peer39_crawler|Perflexity|Perplexity|Petal)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Phind|PiplBot|PoeBot|PoeSearchBot|ProWritingAid|Proximic|Puppeteer|Python\ AI|Qualified|Quark|QuillBot|Qopywriter|Qwen|RAG\ Agent|RAG\ Azure\ AI|RAG\ Chatbot|RAG\ Database|RAG\ IS|RAG\ Pipeline|RAG\ Search)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(RAG\ with|RAG-|RAG_|Raptor|React\ Agent|Redis\ AI\ RAG|RobotSpider|Rytr|SaplingAI|SBIntuitionsBot|Scala|Scalenut|Scrap|ScriptBook|Seekr|SEObot|SEO\ Content\ Machine|SEO\ Robot|SemrushBot|Sentibot)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Serper|ShapBot|Sidetrade|Simplified\ AI|Sitefinity|Skydancer|SlickWrite|SmartBot|Sonic|Sora|Spider/2|SpiderCreator|Spin\ Rewrite|Spinbot|Stability|StableDiffusionBot|Sudowrite|SummalyBot|Super\ Agent|Superagent)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(SuperAGI|Surfer\ AI|TerraCotta|Text\ Blaze|TextCortex|Thinkbot|Thordata|TikTokSpider|Timpibot|Tinybird|Together\ AI|Traefik|TurnitinBot|uAgents|VelenPublicWebCrawler|Venus\ Chub\ AI|Vidnami\ AI|Vision\ RAG|WebSurfer|WebText)") {
-        return 444;
-      }
-      if ($http_user_agent ~* "(Webzio|WeChat|Whisper|WordAI|Wordtune|WPBot|Writecream|WriterZen|Writescope|Writesonic|xAI|xBot|YaML|YandexAdditional|YouBot|Zendesk|Zero|Zhipu|Zhuque\ AI|Zimm)") {
-        return 444;
-      }
-    '';
-  };
-  #greg.proxies."registry.thehellings.com" = {
-  #target = "https://gitea.shire-zebra.ts.net:5000";
-  #ssl = true;
-  #genAliases = false;
-  #extraConfig = ''
-  #proxy_set_header X-Forwarded-Proto https;
-  #proxy_set_header X-Forwarded-Ssl on;
-  #client_max_body_size 25000m;
-  #'';
-  #};
-
-  networking.firewall.allowedTCPPorts = [ sshPort ];
-
-  services = {
-    anubis = {
-      instances = {
-        git = {
-          enable = true;
-          settings = {
-            BIND = "/run/anubis/anubis-git/anubis.sock";
-            COOKIE_DOMAIN = "thehellings.com";
-            SERVE_ROBOTS_TXT = true;
-            TARGET = "https://gitea.shire-zebra.ts.net/";
-          };
-        };
-      };
-    };
-
-    haproxy = {
-      enable = true;
-      config = ''
-        global
-          daemon
-          maxconn 20
-
-        defaults
-          timeout connect 500s
-          timeout client 500s
-          timeout server 1h
-
-        listen gitsshd
-          bind *:${toString sshPort}
-          timeout client 1h
-          mode tcp
-          server git-isaiah isaiah.shire-zebra.ts.net:32222
-          server git-jeremiah jeremiah.shire-zebra.ts.net:32222
-          server git-zeke zeke.shire-zebra.ts.net:32222
-      '';
-    };
-  };
-
-  systemd.services = {
-    haproxy = {
-      after = [
-        "network-online.target"
-      ];
-      wants = [
-        "network-online.target"
-      ];
-    };
-  };
-
-  users.users.nginx.extraGroups = [ config.users.groups.anubis.name ];
-}

hosts/unstable/linode/matrix.nix

@@ -1,70 +0,0 @@
-# Registration of new users is disabled for the public, but I can create
-# them by the following commands:
-# nix run nixpkgs.matrix-synapse
-# register_new_matrix_user -k "B9EoPr2WV9hzwc7uL2Sx1JmvCeKDEOGCpB0uginQcQtEH4wzRtkSIdo7lltrjSQa" http://localhost:8448
-{ config, ... }:
-let
-  domain = "${config.networking.domain}";
-  fqdn = "matrix.${domain}";
-in
-{
-  greg.proxies."${fqdn}" = {
-    extraConfig = ''
-      error_log /var/log/nginx/debug.log debug;
-      proxy_ssl_verify off;
-      proxy_ssl_server_name on;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header X-Forwarded-Ssl on;
-    '';
-    genAliases = false;
-    ssl = true;
-    target = "https://matrix.shire-zebra.ts.net";
-  };
-  services.nginx = {
-    virtualHosts = {
-      # Server the '.well-known' files to find the Matrix API server
-      "${domain}" = {
-        enableACME = true;
-        forceSSL = true;
-        # This is needed so that servers contacting hellings.com can find
-        # the actual application server at matrix.thehellings.com
-        locations."= /.well-known/matrix/server".extraConfig =
-          let
-            server = {
-              "m.server" = "${fqdn}:443";
-            };
-          in
-          ''
-            add_header Content-Type application/json;
-            return 200 '${builtins.toJSON server}';
-          '';
-
-        locations."= /.well-known/matrix/client".extraConfig =
-          let
-            client = {
-              "m.homeserver" = {
-                "base_url" = "https://${fqdn}";
-              };
-              "m.identity_server" = {
-                "base_url" = "https://vector.im";
-              };
-            };
-          in
-          ''
-            add_header Content-Type application/json;
-            add_header Access-Control-Allow-Origin *;
-            return 200 '${builtins.toJSON client}';
-          '';
-      };
-    };
-  };
-
-  # Open networking ports for the server
-  networking.firewall = {
-    enable = true;
-    allowedTCPPorts = [
-      80
-      443
-    ];
-  };
-}

hosts/unstable/linode/nextcloud.nix

@@ -1,58 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-{
-  age.secrets.nextcloudadmin = {
-    file = ../../../secrets/nextcloudadmin.age;
-    owner = "nextcloud";
-  };
-
-  services.nextcloud = {
-    enable = true;
-    package = pkgs.nextcloud33;
-    appstoreEnable = true;
-    hostName = "next.${config.networking.domain}";
-    https = true;
-    config = {
-      adminpassFile = config.age.secrets.nextcloudadmin.path;
-      adminuser = "greg";
-      dbhost = "/run/postgresql";
-      dbtype = "pgsql";
-    };
-    settings = {
-      default_phone_region = "US";
-      overwriteprotocol = "https";
-    };
-  };
-
-  services.nginx.virtualHosts."next.thehellings.com" = {
-    forceSSL = true;
-    enableACME = true;
-  };
-
-  # Otherwise nginx errors looking for the nextcloud sock file
-  systemd.services.nginx.after = [ "nextcloud.service" ];
-
-  greg.backup.jobs.nextcloud-bkup = {
-    src = "/var/lib/nextcloud";
-    dest = "nextcloud-backup";
-    pre = lib.getExe (
-      pkgs.writeShellApplication {
-        name = "nextcloud-backup-pre";
-        runtimeInputs = [ config.services.nextcloud.occ ];
-        text = "nextcloud-occ maintenance:mode --on";
-      }
-    );
-    post = lib.getExe (
-      pkgs.writeShellApplication {
-        name = "nextcloud-backup-post";
-        runtimeInputs = [ config.services.nextcloud.occ ];
-        text = "nextcloud-occ maintenance:mode --off";
-      }
-    );
-  };
-}

hosts/unstable/linode/nginx.nix

@@ -1,38 +0,0 @@
-{ ... }:
-let
-  homepage = "127.0.0.1:30080";
-in
-{
-  security.acme = {
-    acceptTerms = true;
-    defaults.email = "greg.hellings@gmail.com";
-  };
-
-  services.nginx = {
-    enable = true;
-
-    clientMaxBodySize = "25000m"; # To help with uploading container images
-    # If there are recommended settings, let's use them!
-    recommendedGzipSettings = true;
-    recommendedOptimisation = true;
-    recommendedProxySettings = true;
-    recommendedTlsSettings = true;
-  };
-
-  # Actually serve the content from here
-  virtualisation.podman.enable = true;
-  virtualisation.oci-containers = {
-    backend = "podman";
-    containers."homepage" = {
-      image = "registry.thehellings.com:443/greg/homepage/gregs-homepage:latest";
-      ports = [ "${homepage}:80" ];
-    };
-  };
-  greg.proxies = {
-    "thehellings.com" = {
-      target = "http://${homepage}/";
-      ssl = true;
-      genAliases = false;
-    };
-  };
-}

hosts/unstable/linode/podman.nix

@@ -1,13 +0,0 @@
-{ pkgs, ... }:
-
-{
-  environment.systemPackages = with pkgs; [
-    podman-compose
-  ];
-
-  virtualisation.podman = {
-    enable = true;
-    dockerCompat = true;
-    dockerSocket.enable = true;
-  };
-}

hosts/unstable/linode/postgres.nix

@@ -1,63 +0,0 @@
-{
-  config,
-  pkgs,
-  pkgs',
-  ...
-}:
-
-{
-  environment.systemPackages = [ pkgs'.upgrade-pg-cluster ];
-
-  services.postgresql = {
-    enable = true;
-    package = pkgs.postgresql_15;
-    checkConfig = true;
-    ensureDatabases = [ "nextcloud" ];
-    #initialScript = pkgs.writeText "create-matrix-db.sql" ''
-    #	CREATE ROLE "matrix-synapse" WITH LOGIN;
-    #	CREATE DATABASE "synapse" WITH OWNER "matrix-synapse" TEMPLATE template0 LC_COLLATE = "C" LC_CTYPE = "C";
-    #	GRANT ALL PRIVILEGES ON DATABASE "synapse" TO "matrix-synapse";
-    #'';  # These are done manually in order to set the LC_COLLATE values properly
-    ensureUsers = [
-      {
-        name = "nextcloud";
-        ensureDBOwnership = true;
-      }
-    ];
-    settings = {
-      log_connections = true;
-      log_statement = "all";
-      logging_collector = true;
-      log_filename = "postgresql.log";
-    };
-    identMap = ''
-      root root postgres
-    '';
-  };
-
-  services.postgresqlBackup = {
-    enable = true;
-    databases = [ "nextcloud" ];
-  };
-
-  services.logrotate = {
-    enable = true;
-    settings = {
-      postgresBackup = {
-        enable = true;
-        files = "${config.services.postgresqlBackup.location}/*.gz";
-      };
-      postgresLog = {
-        enable = true;
-        files = "/var/lib/postgresql/*/log/*.log";
-        compress = true;
-        compresscmd = "${pkgs.xz}/bin/xz";
-      };
-    };
-  };
-
-  greg.backup.jobs.greg-postgresql-backup = {
-    src = config.services.postgresqlBackup.location;
-    dest = "linode-postgres";
-  };
-}

hosts/unstable/proxmoxtemplate/default.nix

@@ -1,60 +0,0 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
-{ pkgs, ... }:
-
-{
-  imports = [
-    # Include the results of the hardware scan.
-    ./hardware-configuration.nix
-  ];
-
-  # Bootloader.
-  boot.loader = {
-    systemd-boot.enable = true;
-    efi.canTouchEfiVariables = true;
-  };
-
-  environment.systemPackages = with pkgs; [
-  ];
-
-  greg = {
-    home = true;
-    tailscale = {
-      enable = true;
-      tags = [ "home" ];
-    };
-  };
-
-  networking = {
-    hostName = "proxmoxtemplate"; # Define your hostname.
-    # defaultGateway = {
-    #   address = " 10.42.1.2";
-    #   interface = "enp6s18";
-    # };
-    # interfaces = {
-    #   enp6s18 = {
-    #     ipv4.addresses = [
-    #       {
-    #         address = "10.42.1.8";
-    #         prefixLength = 16;
-    #       }
-    #     ];
-    #   };
-    # };
-    nameservers = [ "10.42.1.5" ];
-  };
-
-  services.qemuGuest.enable = true;
-
-  system.stateVersion = "24.11"; # Did you read the comment?
-
-  # Define a user account. Don't forget to set a password with ‘passwd’.
-  users.users.greg = {
-    isNormalUser = true;
-    description = "Greg Hellings";
-    extraGroups = [ "wheel" ];
-    packages = with pkgs; [ ];
-  };
-}

hosts/unstable/proxmoxtemplate/hardware-configuration.nix

@@ -1,50 +0,0 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{
-  lib,
-  modulesPath,
-  ...
-}:
-
-{
-  imports = [
-    (modulesPath + "/profiles/qemu-guest.nix")
-  ];
-
-  boot.initrd.availableKernelModules = [
-    "uhci_hcd"
-    "ehci_pci"
-    "ahci"
-    "virtio_pci"
-    "virtio_scsi"
-    "sd_mod"
-    "sr_mod"
-  ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" = {
-    device = "/dev/disk/by-uuid/507251f1-efe7-448d-8de8-91ee582a9afb";
-    fsType = "ext4";
-  };
-
-  fileSystems."/boot" = {
-    device = "/dev/disk/by-uuid/7115-EFA6";
-    fsType = "vfat";
-    options = [
-      "fmask=0077"
-      "dmask=0077"
-    ];
-  };
-
-  swapDevices = [ ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.enp6s18.useDHCP = lib.mkDefault true;
-}

hosts/unstable/zeke/default.nix

@@ -32,6 +32,10 @@
       enable = true;
       tags = [ "home" ];
     };
+    vmdev = {
+      enable = true;
+      host = "vbox";
+    };
   };
 
   hardware = {
@@ -71,10 +75,4 @@
   users.users.greg.extraGroups = [
     "podman"
   ];
-
-  # virtualisation.virtualbox.host = {
-  #   enableExtensionPack = true;
-  #   headless = true;
-  #   enableWebService = true;
-  # };
 }

manifests/donetick/configmap.yaml

@@ -1,79 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: donetick-config
-  namespace: donetick
-data:
-  # Value pulled from
-  # https://github.com/donetick/donetick/blob/main/config/selfhosted.yaml
-  selfhosted.yaml: |-
-    name: "selfhosted"
-    is_done_tick_dot_com: false
-    is_user_creation_disabled: false
-    telegram:
-      token: ""
-    pushover:
-      token: ""
-    database:
-      type: "sqlite"
-      migration: true
-      # these are only required for postgres
-      host: "secret"
-      port: 5432
-      user: "secret"
-      password: "secret"
-      name: "secret"
-    jwt:
-      secret: "This is really a secure JWT secret now!"
-      session_time: 168h
-      max_refresh: 168h
-    server:
-      port: 2021
-      read_timeout: 10s
-      write_timeout: 10s
-      rate_period: 60s
-      rate_limit: 300
-      cors_allow_origins:
-        - "http://localhost:5173"
-        - "http://localhost:7926"
-        # the below are required for the android app to work
-        - "https://localhost"
-        - "capacitor://localhost"
-      serve_frontend: true
-    logging:
-      level: "info"
-      encoding: "json"
-      development: false
-    scheduler_jobs:
-      due_job: 30m
-      overdue_job: 3h
-      pre_due_job: 3h
-    email:
-      host:
-      port:
-      key:
-      email:
-      appHost:
-    oauth2:
-      client_id:
-      client_secret:
-      auth_url:
-      token_url:
-      user_info_url:
-      redirect_url:
-      name:
-    # Real-time configuration
-    realtime:
-      enabled: true
-      sse_enabled: true
-      heartbeat_interval: 60s
-      connection_timeout: 120s
-      max_connections: 1000
-      max_connections_per_user: 5
-      event_queue_size: 2048
-      cleanup_interval: 2m
-      stale_threshold: 5m
-      enable_compression: true
-      enable_stats: true
-      allowed_origins:
-        - "*"

manifests/donetick/deployment.yaml

@@ -1,38 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: donetick
-  namespace: donetick
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: donetick
-  template:
-    metadata:
-      labels:
-        app: donetick
-    spec:
-      containers:
-        - name: donetick
-          image: donetick/donetick
-          ports:
-            - containerPort: 2021
-              name: http
-          env:
-            - name: DT_ENV
-              value: "selfhosted"
-            - name: DT_SQLITE_PATH
-              value: "/data/donetick.db"
-          volumeMounts:
-            - name: config
-              mountPath: /config
-            - name: data
-              mountPath: /data
-      volumes:
-        - name: config
-          configMap:
-            name: donetick-config
-        - name: data
-          persistentVolumeClaim:
-            claimName: donetick-data

manifests/donetick/ingress.yaml

@@ -1,15 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: donetick-tailscale
-  namespace: donetick
-spec:
-  ingressClassName: tailscale
-  defaultBackend:
-    service:
-      name: donetick
-      port:
-        number: 2021
-  tls:
-    - hosts:
-        - todo

manifests/donetick/kustomization.yaml

@@ -1,9 +0,0 @@
-namespace: donetick
-
-resources:
-  - namespace.yaml
-  - configmap.yaml
-  - pvc.yaml
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml

manifests/donetick/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: donetick

manifests/donetick/pvc.yaml

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: donetick-data
-  namespace: donetick
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 5Gi

manifests/donetick/service.yaml

@@ -1,13 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: donetick
-  namespace: donetick
-spec:
-  selector:
-    app: donetick
-  ports:
-    - name: http
-      port: 2021
-      targetPort: 2021
-      protocol: TCP

manifests/gitea-runner/chart.yaml

@@ -1,56 +0,0 @@
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: gitea
-spec:
-  interval: "24h"
-  url: https://dl.gitea.com/charts/
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: gitea-runner
-  namespace: gitea-runner
-spec:
-  interval: 10m
-  chart:
-    spec:
-      chart: actions
-      version: "0.0.4"
-      sourceRef:
-        kind: HelmRepository
-        name: gitea
-      interval: "1h"
-  values:
-    rbac:
-      create: true
-    serviceAccount:
-      create: true
-    gitea:
-      instanceURL: https://src.thehellings.com
-      runnerToken:
-        existingSecret: gitea-runner
-        existingSecretKey: token
-    imagePullSecrets:
-      - name: image-pull-secrets
-    config:
-      runner:
-        labels:
-          # Ubuntu
-          - "ubuntu-22.04:docker://ubuntu:22.04"
-          - "ubuntu-24.04:docker://ubuntu:24.04"
-          - "ubuntu-24.10:docker://ubuntu:24.10"
-          # Fedora
-          - "fedora-41:docker://fedora:41"
-          - "fedora-42:docker://fedora:42"
-          # CentOS Stream
-          - "centos-stream-9:docker://quay.io/centos/centos:stream9"
-          - "centos-stream-10:docker://quay.io/centos/centos:stream10"
-          # Nix
-          - "nix:docker://nixos/nix:latest"
-          # ci-images (internal registry: src.thehellings.com/greg)
-          - "ci-builder:docker://src.thehellings.com/greg/builder:latest"
-          - "ci-vm-test:docker://src.thehellings.com/greg/vm-test:latest"
-          - "ci-sword:docker://src.thehellings.com/greg/sword-container-builder:latest"
-          - "ci-bitwarden:docker://src.thehellings.com/greg/bitwarden:latest"
-          - "ci-immich:docker://src.thehellings.com/greg/immich:latest"

manifests/gitea-runner/kustomization.yaml

@@ -1,6 +0,0 @@
-namespace: gitea-runner
-
-resources:
-  - namespace.yaml
-  - secrets.yaml
-  - chart.yaml

manifests/gitea-runner/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: gitea-runner

manifests/gitea-runner/secrets.yaml

@@ -1,18 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: gitea-runner
-  namespace: gitea-runner
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    name: bitwarden-login
-    kind: ClusterSecretStore
-  target:
-    name: gitea-runner
-    creationPolicy: Owner
-  data:
-    - secretKey: token
-      remoteRef:
-        key: 11419680-5338-4f19-bdd9-b422007046af
-        property: password

manifests/gitea/chart.yaml

@@ -15,7 +15,7 @@ spec:
   chart:
     spec:
       chart: gitea
-      version: "12.6.0"
+      version: "12.7.0"
       sourceRef:
         kind: HelmRepository
         name: gitea-repository
@@ -35,7 +35,7 @@ spec:
       storageClass: longhorn-default
 
     image:
-      tag: "1.26.2"
+      tag: "1.27.1"
 
     replicaCount: 1
 
@@ -67,7 +67,7 @@ spec:
           RUN_MODE: dev
         server:
           DOMAIN: "shire-zebra.ts.net"
-          ROOT_URL: "https://gitea.shire-zebra.ts.net"
+          ROOT_URL: "https://git.k3s.thehellings.lan"
           SSH_PORT: "2222"
         database:
           DB_TYPE: postgres
@@ -85,15 +85,15 @@ spec:
           DISABLE_REGISTRATION: "true"
         storage:
           STORAGE_TYPE: minio
-          MINIO_ENDPOINT: "nas1.shire-zebra.ts.net:9000"
+          MINIO_ENDPOINT: "nas1.shire-zebra.ts.net:30188"
           MINIO_BUCKET: gitea
-          MINIO_LOCATION: us-east-1
+          MINIO_LOCATION: garage
           # MINIO_ACCESS_KEY_ID: ""
           # MINIO_SECRET_ACCESS_KEY: ""
           MINIO_USE_SSL: "false"
           MINIO_INSECURE_SKIP_VERIFY: "true"
-        webhook:
+        security:
-          ALLOWED_HOST_LIST: loopback,private,*.shire-zebra.ts.net
+          ALLOWED_HOST_LIST: loopback,private,*.shire-zebra.ts.net,*.nebula.thehellings.com,*.thehellings.lan
 
       metrics:
         enabled: false
@@ -102,8 +102,8 @@ spec:
 
     persistence:
       enabled: true
-      storageClass: longhorn-default
+      create: false
-      size: "50Gi"
+      claimName: gitea-new
 
     # I will manage my Postgres externally
     postgresql:

manifests/gitea/dump-cronjob.yaml

@@ -18,12 +18,12 @@ spec:
           volumes:
             - name: gitea-data
               persistentVolumeClaim:
-                claimName: gitea-shared-storage
+                claimName: gitea-new
             - name: dump-staging
               emptyDir: {}
           initContainers:
             - name: gitea-dump
-              image: "gitea/gitea:1.25.4"
+              image: "gitea/gitea:1.27.1"
               command:
                 - /bin/sh
                 - "-c"
@@ -51,12 +51,12 @@ spec:
                 - |
                   set -e
                   # Configure mc alias for MinIO
-                  mc alias set nas1 http://nas1.shire-zebra.ts.net:9000 \
+                  mc alias set nas1 http://nas1.shire-zebra.ts.net:30188 \
                     "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
                   # Upload dump to backup-gitea bucket
                   DUMP_FILE=$(ls /dump-staging/gitea-dump-*.zip | head -1)
-                  mc cp "${DUMP_FILE}" "nas1/backup-gitea/$(basename ${DUMP_FILE})"
+                  mc cp "${DUMP_FILE}" "nas1/gitea-backup/$(basename ${DUMP_FILE})"
-                  echo "Uploaded $(basename ${DUMP_FILE}) to backup-gitea"
+                  echo "Uploaded $(basename ${DUMP_FILE}) to gitea-backup"
                   # Set 30-day lifecycle on the bucket (idempotent)
                   mc ilm rule add --expire-days 30 nas1/backup-gitea 2>/dev/null || true
               volumeMounts:
@@ -69,10 +69,10 @@ spec:
                 - name: MINIO_ACCESS_KEY
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-config
+                      name: gitea-backup
                       key: minio_key
                 - name: MINIO_SECRET_KEY
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-config
+                      name: gitea-backup
                       key: minio_secret

manifests/gitea/ingress.yaml

@@ -12,3 +12,20 @@ spec:
   tls:
     - hosts:
         - gitea
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gitea-direct
+spec:
+  rules:
+    - host: git.k3s.thehellings.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gitea-release-http
+                port:
+                  name: http

manifests/gitea/secrets.yaml

@@ -1,5 +1,32 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
+metadata:
+  name: gitea-backup
+spec:
+  target:
+    name: gitea-backup
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        minio_key: "{{ .minio_key }}"
+        minio_secret: "{{ .minio_secret }}"
+  secretStoreRef:
+    name: bitwarden-login
+    kind: ClusterSecretStore
+  data:
+    # MinIO credentials
+    - secretKey: minio_key
+      remoteRef:
+        key: dfb2f0c8-110d-4e96-83a7-b49c001c0897
+        property: username
+    - secretKey: minio_secret
+      remoteRef:
+        key: dfb2f0c8-110d-4e96-83a7-b49c001c0897
+        property: password
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
 metadata:
   name: gitea-config
 spec:
@@ -21,23 +48,14 @@ spec:
     name: bitwarden-login
     kind: ClusterSecretStore
   data:
-    # MinIO credentials
-    - secretKey: minio_key
-      remoteRef:
-        key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d
-        property: username
-    - secretKey: minio_secret
-      remoteRef:
-        key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d
-        property: password
     # MinIO credentials for NAS1
     - secretKey: minio_nas1_key
       remoteRef:
-        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
+        key: 33e8e4e0-eb90-484c-9ec9-b3a8018077a3
         property: username
     - secretKey: minio_nas1_secret
       remoteRef:
-        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
+        key: 33e8e4e0-eb90-484c-9ec9-b3a8018077a3
         property: password
     # Postgres credentials
     - secretKey: dbuser

manifests/helm/longhorn.yaml

@@ -27,7 +27,7 @@ spec:
   chart:
     spec:
       chart: longhorn
-      version: "1.11.2"
+      version: "1.11.3"
       sourceRef:
         kind: HelmRepository
         name: longhorn
@@ -141,6 +141,10 @@ spec:
                   number: 80
     - <<: *host
       host: longhorn.kubernetes
+    - <<: *host
+      host: longhorn.k3s.nebula.thehellings.com
+    - <<: *host
+      host: longhorn.k3s.thehellings.lan
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass

manifests/helm/traefik.yaml

@@ -10,6 +10,16 @@ spec:
       - "--api.dashboard=true"
       - "--api.insecure=true"
       - "--log.level=DEBUG"
+      # Access logging: gives per-request visibility (client IP, host,
+      # path, bytes, duration) for every ingress route Traefik terminates
+      # (git.k3s.thehellings.lan, matrix.k3s.thehellings.lan, immich, etc).
+      # This is the layer HAProxy on linode forwards :80 traffic to, so
+      # having request-level logs here is essential for tracing bandwidth
+      # spikes back to a specific host/path/client rather than just a
+      # backend-level byte count.
+      - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.fields.headers.defaultmode=keep"
     ports:
       postgres:
         expose:
@@ -20,3 +30,4 @@ spec:
       traefik:
         expose:
           default: true
+

manifests/immich/backup.yaml

@@ -33,24 +33,24 @@ spec:
         access-key: "{{ .minio_key }}"
         secret-key: "{{ .minio_secret }}"
         rclone.conf: |
-          [nas1minio]
+          [garage]
           type = s3
           provider = Minio
-          endpoint = http://nas1.shire-zebra.ts.net:9000
+          endpoint = http://nas1.shire-zebra.ts.net:30188
           access_key_id = {{ .minio_key }}
           secret_access_key = {{ .minio_secret }}
-          region = us-east-1
+          region = garage
   secretStoreRef:
     name: bitwarden-login
     kind: ClusterSecretStore
   data:
     - secretKey: minio_key
       remoteRef:
-        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
+        key: 8fce2750-aa62-4892-b90c-b49c001f494b
         property: username
     - secretKey: minio_secret
       remoteRef:
-        key: c4c66ab3-2ade-4086-9c0d-b3a80172b1ba
+        key: 8fce2750-aa62-4892-b90c-b49c001f494b
         property: password
 ---
 apiVersion: v1
@@ -128,7 +128,7 @@ spec:
                     --progress \
                     --transfers 4 \
                     --checkers 8 \
-                    /staging nas1minio:immich
+                    /staging garage:immich
               volumeMounts:
                 - name: staging
                   mountPath: /staging

manifests/immich/chart.yaml

@@ -32,7 +32,7 @@ spec:
         containers:
           main:
             image:
-              tag: v2.7.5
+              tag: v3.1.0
             env:
               DB_HOSTNAME: immich-rw
               DB_DATABASE_NAME: immich

manifests/immich/ingress.yaml

@@ -22,6 +22,10 @@ spec:
                 name: immich-server
                 port:
                   name: http
+    - <<: *host
+      host: immich.k3s.nebula.thehellings.com
+    - <<: *host
+      host: immich.k3s.thehellings.lan
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress

manifests/kustomization.yaml

@@ -10,7 +10,4 @@ resources:
   - immich
   - monitoring
   - pinchflat
-  - smokeping
-  - uptimekuma
-  - donetick
   - gitea

manifests/matrix/ingress.yaml

@@ -13,3 +13,20 @@ spec:
   tls:
     - hosts:
         - matrix
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: matrix-direct
+spec:
+  rules:
+    - host: matrix.k3s.thehellings.lan
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: dendrite
+                port:
+                  number: 8008

manifests/monitoring/config.yaml

@@ -51,8 +51,8 @@ data:
         static_configs:
           - targets:
               - "10.42.0.3"    # OpenWRT access point
-              - "10.42.0.4"    # Joel (Proxmox)
+              - "10.42.0.4"    # pve1 (Proxmox)
-              - "10.42.1.1"    # pve1 (Proxmox)
+              - "10.42.1.1"    # UDM gateway (Ubiquiti)
               - "10.42.1.4"    # chronicles (Synology NAS)
               - "10.42.1.14"   # nas1 (TrueNAS)
               - "10.42.2.57"   # odoo

manifests/smokeping/deployment.yaml

@@ -1,50 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: smokeping
-  labels:
-    app: smokeping
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: smokeping
-  template:
-    metadata:
-      labels:
-        app: smokeping
-    spec:
-      containers:
-        - name: smokeping
-          image: docker.io/linuxserver/smokeping:2.9.0
-          imagePullPolicy: IfNotPresent
-          ports:
-            - name: http
-              containerPort: 80
-              protocol: TCP
-          volumeMounts:
-            - name: config
-              mountPath: /config
-            - name: data
-              mountPath: /data
-          env:
-            - name: PUID
-              value: "1000"
-            - name: PGID
-              value: "1000"
-            - name: TZ
-              value: "America/Chicago"
-            #- name: MASTER_URL
-            #  value: "https://ping.shire-zebra.ts.net"
-            # SHARED_SECRET if you want to run a cluster
-            # CACHE_DIR if you need to explicitly state that
-      restartPolicy: Always
-      volumes:
-        - name: config
-          persistentVolumeClaim:
-            claimName: smokeping-config
-        - name: data
-          persistentVolumeClaim:
-            claimName: smokeping-data

manifests/smokeping/ingress.yaml

@@ -1,14 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: smokeping-tailscale
-spec:
-  ingressClassName: tailscale
-  defaultBackend:
-    service:
-      name: smokeping
-      port:
-        name: http
-  tls:
-    - hosts:
-        - ping

manifests/smokeping/kustomization.yaml

@@ -1,8 +0,0 @@
-namespace: smokeping
-
-resources:
-  - namespace.yaml
-  - pvc.yaml
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml

manifests/smokeping/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: smokeping

manifests/smokeping/pvc.yaml

@@ -1,23 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: smokeping-config
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-default
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: smokeping-data
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-default
-  resources:
-    requests:
-      storage: 25Gi

manifests/smokeping/service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: smokeping
-  labels:
-    app: smokeping
-spec:
-  type: ClusterIP
-  ports:
-    - port: 80
-      targetPort: http
-      protocol: TCP
-      name: http
-  selector:
-    app: smokeping

manifests/uptimekuma/chart.yaml

@@ -1,38 +0,0 @@
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: uptime-kuma
-  namespace: uptime-kuma
-spec:
-  interval: "24h"
-  url: "https://helm.irsigler.cloud"
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: uptime-kuma
-  namespace: uptime-kuma
-spec:
-  interval: 10m
-  chart:
-    spec:
-      chart: uptime-kuma
-      sourceRef:
-        kind: HelmRepository
-        name: uptime-kuma
-      interval: "1h"
-  dependsOn:
-    - name: longhorn
-      namespace: longhorn-system
-    - name: mariadb-cluster
-      namespace: mariadb-operator
-  values:
-    volume:
-      storageClassName: longhorn-default
-    image:
-      tag: "2.0.2"
-    externalDatabase:
-      enabled: true
-      hostname: mariadb-cluster.mariadb-operator.svc.cluster.local
-      database: uptimekuma
-      existingSecret: uptimekuma-mariadb-password

manifests/uptimekuma/ingress.yaml

@@ -1,15 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: uptime-kuma-tailscale
-  namespace: uptime-kuma
-spec:
-  ingressClassName: tailscale
-  defaultBackend:
-    service:
-      name: uptime-kuma
-      port:
-        number: 3001
-  tls:
-    - hosts:
-        - kuma

manifests/uptimekuma/kustomization.yaml

@@ -1,7 +0,0 @@
-namespace: uptimekuma
-
-resources:
-  - namespace.yaml
-  - secrets.yaml
-  - chart.yaml
-  - ingress.yaml

manifests/uptimekuma/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: uptimekuma

manifests/uptimekuma/secrets.yaml

@@ -1,27 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: uptimekuma-mariadb-password
-spec:
-  target:
-    name: uptimekuma-mariadb-password
-    deletionPolicy: Delete
-    template:
-      type: kubernetes.io/basic-auth
-      data:
-        username: |-
-          {{ .username }}
-        password: |-
-          {{ .password }}
-  secretStoreRef:
-    name: bitwarden-login
-    kind: ClusterSecretStore
-  data:
-    - secretKey: username
-      remoteRef:
-        key: 4df95656-9f9c-4916-8e34-b3a200376365
-        property: username
-    - secretKey: password
-      remoteRef:
-        key: 4df95656-9f9c-4916-8e34-b3a200376365
-        property: password

modules/hm/gui/bookmarks.nix

@@ -230,7 +230,7 @@
             bookmarks = [
               {
                 name = "PVE1";
-                url = "https://10.42.1.1:8006/";
+                url = "https://10.42.0.4:8006/";
               }
               {
                 name = "Jeremiah";

modules/nix-conf.nix

@@ -62,12 +62,12 @@ in
             if cfg.cache then
               [
                 #"http://chronicles.shire-zebra.ts.net:9000/binary-cache/"
-                "http://nas1.shire-zebra.ts.net:9000/niks3"
+                "http://niks3.nas1.shire-zebra.ts.net:30189/"
                 #"http://nas1.shire-zebra.ts.net:8080/default"
               ]
             else
               [
-                "http://nas1.thehellings.lan:8080/default"
+                "http://niks3.nas1.thehellings.lan:30189/"
               ]
           )
           ++ [

modules/nixos/ceph.nix

@@ -1,927 +0,0 @@
-# This is a good source for a Ceph dealio
-# https://gist.github.com0/nh2/13425a1f18b4c1ce82edb63c10b163c9
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-
-with lib;
-
-let
-  cfg = config.services.ceph-benaco;
-  commaSep = builtins.concatStringsSep ",";
-
-  ensureUnitExists =
-    c': name:
-    let
-    in
-    #unitName = (builtins.elemAt (builtins.split "\\." name) 0);
-    if c'.systemd.services ? unitName then name else name; # "Unable to locate ${name} at ${commaSep (builtins.attrNames c')}";
-in
-
-{
-
-  ###### interface
-
-  options = {
-
-    services.ceph-benaco = {
-
-      enable = mkEnableOption "Ceph distributed filesystem";
-
-      package = mkOption {
-        type = types.package;
-        default = pkgs.ceph;
-        defaultText = literalExpression "pkgs.ceph-benaco";
-        description = "Ceph package to use.";
-      };
-
-      fsid = mkOption {
-        type = types.str;
-        description = "Unique cluster identifier.";
-      };
-
-      clusterName = mkOption {
-        type = types.str;
-        description = "Cluster name.";
-        default = "ceph";
-      };
-
-      initialMonitors = mkOption {
-        type = types.listOf (
-          types.submodule {
-            options = {
-              hostname = mkOption {
-                type = types.str;
-                description = "Initial monitor hostname.";
-              };
-
-              ipAddress = mkOption {
-                type = types.str;
-                description = "Initial monitor IP address.";
-              };
-            };
-          }
-        );
-        description = "Initial monitors.";
-      };
-
-      mdsNodes = mkOption {
-        type = types.listOf (
-          types.submodule {
-            options = {
-              hostname = mkOption {
-                type = types.str;
-                description = "MDS hostname.";
-              };
-
-              ipAddress = mkOption {
-                type = types.str;
-                description = "MDS IP address.";
-              };
-            };
-          }
-        );
-        description = "MDS nodes.";
-      };
-
-      publicNetworks = mkOption {
-        type = types.listOf types.str;
-        description = "Public network(s) of the cluster.";
-      };
-
-      clusterNetworks = mkOption {
-        type = types.listOf types.str;
-        description = "Cluster backend networks for OSD sync";
-      };
-
-      adminKeyring = mkOption {
-        type = types.path;
-        description = "Ceph admin keyring to install on the machine.";
-      };
-
-      monitor = {
-        enable = mkEnableOption "Activate a Ceph monitor on this machine.";
-
-        initialKeyring = mkOption {
-          type = types.path;
-          description = "Keyring file to use when initializing a new monitor";
-          example = "/path/to/ceph.mon.keyring";
-        };
-
-        nodeName = mkOption {
-          type = types.str;
-          description = "Ceph monitor node name.";
-          example = "node1";
-        };
-
-        bindAddr = mkOption {
-          type = types.str;
-          description = "IP address that the OSDs shall bind to.";
-          example = "10.0.0.1";
-        };
-
-        advertisedPublicAddr = mkOption {
-          type = types.str;
-          description = "IP address that the monitor shall advertise.";
-          example = "10.0.0.1";
-        };
-      };
-
-      manager = {
-        enable = mkEnableOption "Activate a Ceph manager on this machine.";
-
-        nodeName = mkOption {
-          type = types.str;
-          description = "Ceph manager node name.";
-          example = "node1";
-        };
-      };
-
-      osdBindAddr = mkOption {
-        type = types.str;
-        description = "IP address that the OSDs shall bind to.";
-        example = "10.0.0.1";
-      };
-
-      osdAdvertisedPublicAddr = mkOption {
-        type = types.str;
-        description = "IP address that the OSDs shall advertise.";
-        example = "10.0.0.1";
-      };
-
-      osds = mkOption {
-        default = { };
-        example = {
-          osd1 = {
-            enable = true;
-            bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
-            id = 1;
-            uuid = "11111111-1111-1111-1111-111111111111";
-            blockDevice = "/dev/sdb";
-            blockDeviceUdevRuleMatcher = ''KERNEL=="sdb"'';
-            clusterAddress = "10.1.0.1";
-          };
-          osd2 = {
-            enable = true;
-            bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
-            id = 2;
-            uuid = "22222222-2222-2222-2222-222222222222";
-            blockDevice = "/dev/sdc";
-            blockDeviceUdevRuleMatcher = ''KERNEL=="sdc"'';
-            clusterAddress = "10.1.0.2";
-          };
-        };
-        description = ''
-          This option allows you to define multiple Ceph OSDs.
-          A common idiom is to use one OSD per physical hard drive.
-
-          Note that the OSD names given as attributes of this key
-          are NOT what ceph calls OSD IDs (instead, those are defined
-          by the 'services.ceph-benaco.osds.*.id' fields).
-          Instead, the name is an identifier local and unique to the
-          current machine only, used only to name the systemd service
-          for that OSD.
-        '';
-        type = types.attrsOf (
-          types.submodule {
-            options = {
-
-              enable = mkEnableOption "Activate a Ceph OSD on this machine.";
-
-              bootstrapKeyring = mkOption {
-                type = types.path;
-                description = "Ceph OSD bootstrap keyring.";
-                example = "/path/to/ceph.client.bootstrap-osd.keyring";
-              };
-
-              id = mkOption {
-                type = types.int;
-                description = "The ID of this OSD. Must be unique in the Ceph cluster.";
-                example = 1;
-              };
-
-              uuid = mkOption {
-                type = types.str;
-                description = "The UUID of this OSD. Must be unique in the Ceph cluster.";
-                example = "abcdef12-abcd-1234-abcd-1234567890ab";
-              };
-
-              systemdExtraRequiresAfter = mkOption {
-                type = types.listOf types.str;
-                default = [ ];
-                description = ''
-                  Add the specified systemd units to the "requires" and "after"
-                  lists of the systemd service of this OSD.
-
-                  Useful, for example, to decrypt the underlying block devices with LUKS first.
-
-                  NixOS modules allow override those lists from outside, but for that
-                  the names of the systemd services for the OSDs need to be known;
-                  this option is a convenience to not have to know them from outside.
-                '';
-                example = "decrypt-my-disk.service";
-              };
-
-              skipZap = mkOption {
-                type = types.bool;
-                default = false;
-                description = ''
-                  Whether to skip the zapping of the the OSD device on initial OSD
-                  installation.
-
-                  Skipping is needed because <command>ceph-volume</command> cannot
-                  zap device-mapper devices:
-                  <link xlink:href="https://tracker.ceph.com/issues/24504" />
-
-                  In that case you need to wipe the device manually.
-
-                  In the common case of placing the OSD on a cryptsetup LUKS device
-                  (which is a device-mapper device), re-creating the encryption
-                  from scratch with a new key zaps anything anyway, in which case
-                  zapping can be skipped here.
-                '';
-              };
-
-              blockDevice = mkOption {
-                type = types.str;
-                description = "The block device used to store the OSD.";
-                example = "/dev/sdb";
-              };
-
-              blockDeviceUdevRuleMatcher = mkOption {
-                type = types.str;
-                description = ''
-                  An udev rule matcher matching the block device used to store the OSD.
-                  Will be spliced into the udev rule that is
-                  used to set access permissions to the ceph user via an udev rule.
-
-                  This is a matcher instead of just a device name to allow flexibility:
-                  Normal disks can be easily matched with <code>KERNEL=="sda1"</code>, but
-                  device-mapper may not; for example, decrypted cryptsetup LUKS devices
-                  have a less useful <code>KERNEL=="dm-4"</code> and may better be matched
-                  using <code>ENV{DM_NAME}=="mydisk-decrypted"</code>.
-                '';
-                example = ''KERNEL=="sdb"'';
-              };
-
-              dbBlockDevice = mkOption {
-                type = types.nullOr types.str;
-                default = null;
-                description = ''
-                  The block device used to store the OSD's BlueStore DB device.
-
-                  Put this on a faster device than <option>blockDevice</option> to improve performance.
-
-                  See <link xlink:href="http://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/" />
-                  for details.
-                '';
-                example = "/dev/sdc";
-              };
-
-              dbBlockDeviceUdevRuleMatcher = mkOption {
-                type = types.nullOr types.str;
-                default = null;
-                description = ''
-                  Like <option>blockDeviceUdevRuleMatcher</option> but for the
-                  <option>dbBlockDevice</option>.
-                '';
-                example = ''KERNEL=="sdc"'';
-              };
-
-              clusterAddress = mkOption {
-                type = types.nullOr types.str;
-                default = null;
-                description = ''
-                  The IP address on the dedicated cluster network that
-                  is used by the backend communication for OSD communication.
-                '';
-                example = "10.1.0.1f";
-              };
-
-            };
-          }
-        );
-      };
-
-      mds = {
-        enable = mkEnableOption "Activate a Ceph MDS on this machine.";
-
-        nodeName = mkOption {
-          type = types.str;
-          description = "Ceph MDS node name.";
-          example = "node1";
-        };
-
-        listenAddr = mkOption {
-          type = types.str;
-          description = "IP address that the MDS shall advertise.";
-          example = "10.0.0.1";
-        };
-      };
-
-      extraConfig = mkOption {
-        type = types.str;
-        default = "";
-        description = ''
-          Additional ceph.conf settings.
-
-          See the sample file for inspiration:
-          <link xlink:href="https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf" />
-        '';
-      };
-    };
-  };
-
-  ###### implementation
-
-  config =
-    let
-      monDir = "/var/lib/ceph/mon/${cfg.clusterName}-${cfg.monitor.nodeName}";
-      mgrDir = "/var/lib/ceph/mgr/${cfg.clusterName}-${cfg.manager.nodeName}";
-      mdsDir = "/var/lib/ceph/mds/${cfg.clusterName}-${cfg.mds.nodeName}";
-
-      # File permissions for things that are on locations wiped at start
-      # (e.g. /run or its /var/run symlink).
-      ensureTransientCephDirs = ''
-        install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
-      '';
-
-      # File permissions from cluster deployed with ceph-deploy.
-      ensureCephDirs = ''
-        install -m 3770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/log/ceph
-        install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
-        install -m 750 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph
-        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mon
-        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mgr
-        install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/osd
-      '';
-
-      # Utilities called by Ceph device health scraping, see:
-      # https://docs.ceph.com/en/latest/rados/operations/devices/#enabling-monitoring
-      # As per https://github.com/ceph/ceph-container/pull/1490/commits/c49e821599965ae92a88b2c78077ee03c4405895,
-      # both the OSDs and the `mon` need this.
-      # Ceph calls these utilities with `sudo`. That requires sudoers entries.
-      # Sudoers entries require absolute path; that exact (nix store) path needs to
-      # be used by Ceph, so it needs to be given to the systemd unit via `path`.
-      # This is why we pair each `sudoersExtraRule` with the `package` to put onto
-      # that `path`.
-      #
-      # Entries are based on:
-      # https://github.com/ceph/ceph/blob/a2f5a3c1dbfa4dce41e25da4f029a8fdb8c8d864/sudoers.d/ceph-smartctl
-      cephMonitoringSudoersCommandsAndPackages = [
-        {
-          package = pkgs.smartmontools;
-          sudoersExtraRule = {
-            # entry for `security.sudo.extraRules`
-            users = [ config.users.users.ceph.name ];
-            commands = [
-              {
-                command = "${lib.getBin pkgs.smartmontools}/bin/smartctl -x --json=o /dev/*";
-                options = [ "NOPASSWD" ];
-              }
-            ];
-          };
-        }
-        {
-          package = pkgs.nvme-cli;
-          sudoersExtraRule = {
-            # entry for `security.sudo.extraRules`
-            users = [ config.users.users.ceph.name ];
-            commands = [
-              {
-                command = "${lib.getBin pkgs.nvme-cli}/bin/nvme * smart-log-add --json /dev/*";
-                options = [ "NOPASSWD" ];
-              }
-            ];
-          };
-        }
-      ];
-
-      cephDeviceHealthMonitoringPathsOrPackages =
-        with pkgs;
-        [
-          # Contains `sudo`. Ceph wraps this around the other health check programs.
-          # Cannot use `pkgs.sudo` because that one is not SUID, see:
-          # https://discourse.nixos.org/t/sudo-uid-issues/9133
-          "/run/wrappers" # `systemd.services.<name>.path` adds the `bin/` subdir of this
-        ]
-        ++ map ({ package, ... }: package) cephMonitoringSudoersCommandsAndPackages;
-
-      # Unused localOsdServiceName in the following line
-      # deadnix: skip
-      makeCephOsdSetupSystemdService =
-        _localOsdServiceName: osdConfig:
-        let
-          osdExistenceFile = "/var/lib/ceph/osd/.${toString osdConfig.id}.${osdConfig.uuid}.nix-existence";
-        in
-        mkIf osdConfig.enable {
-          description = "Initialize Ceph OSD";
-
-          requires = osdConfig.systemdExtraRequiresAfter;
-          after = osdConfig.systemdExtraRequiresAfter;
-
-          path = with pkgs; [
-            # The following are currently missing in Ceph's wrapping, see https://github.com/NixOS/nixpkgs/issues/147801#issue-1065600852
-            util-linux # for `lsblk`
-            lvm2 # for `lvs`
-          ];
-
-          # TODO Use `udevadm trigger --settle` instead of the separate `udevadm settle`
-          #      once that feature is available to us with systemd >= 238;
-          #      see https://github.com/systemd/systemd/commit/792cc203a67edb201073351f5c766fce3d5eab45
-          preStart = ''
-            set -x
-            ${ensureCephDirs}
-            install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/bootstrap-osd
-            # `install` is not atomic, see
-            # https://lists.gnu.org/archive/html/bug-coreutils/2010-02/msg00243.html
-            # so use `mktemp` + `mv` to make it atomic.
-            TMPFILE=$(mktemp --tmpdir=/var/lib/ceph/bootstrap-osd/)
-            install -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} ${osdConfig.bootstrapKeyring} "$TMPFILE"
-            mv "$TMPFILE" /var/lib/ceph/bootstrap-osd/ceph.keyring
-
-            # Trigger udev rules for permissions of block devices and wait for them to settle.
-            udevadm trigger --name-match=${osdConfig.blockDevice}
-          ''
-          + lib.optionalString (osdConfig.dbBlockDevice != null) ''
-            udevadm trigger --name-match=${osdConfig.dbBlockDevice}
-          ''
-          + ''
-            udevadm settle
-          ''
-          + (optionalString (!osdConfig.skipZap) (
-            ''
-              # Zap OSD block devices, otherwise `ceph-osd` below will try to fsck if there's some old
-              # ceph data on the block device (see https://tracker.ceph.com/issues/24099).
-              ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.blockDevice}
-            ''
-            + lib.optionalString (osdConfig.dbBlockDevice != null) ''
-              ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.dbBlockDevice}
-            ''
-          ));
-
-          script = ''
-            set -euo pipefail
-            set -x
-            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
-            do
-              sleep 1
-            done
-
-            OSD_SECRET=$(${cfg.package}/bin/ceph-authtool --gen-print-key)
-            echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
-              ${cfg.package}/bin/ceph --cluster ${cfg.clusterName} osd new ${osdConfig.uuid} ${toString osdConfig.id} -i - \
-              -n client.bootstrap-osd -k ${osdConfig.bootstrapKeyring}
-            mkdir -p /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}
-
-            ln -s ${osdConfig.blockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block
-          ''
-          + lib.optionalString (osdConfig.dbBlockDevice != null) ''
-            ln -s ${osdConfig.dbBlockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block.db
-          ''
-          + ''
-
-            ${cfg.package}/bin/ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${toString osdConfig.id}/keyring \
-              --name osd.${toString osdConfig.id} --add-key $OSD_SECRET
-
-            ${cfg.package}/bin/ceph-osd -i ${toString osdConfig.id} --mkfs --osd-uuid ${osdConfig.uuid} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} --osd-objectstore bluestore
-            touch ${osdExistenceFile}
-          '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph, preStart as root
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${osdExistenceFile}";
-          };
-        };
-
-      makeCephOsdSystemdService =
-        localOsdServiceName: osdConfig:
-        mkIf osdConfig.enable {
-          description = "Ceph OSD";
-
-          # Note we do not have to add `osdConfig.systemdExtraRequiresAfter` here because
-          # that's already a dependency of our dependency `ceph-osd-setup-*`.
-          requires = [ (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          path = [
-            # TODO: use wrapProgram in the ceph package for this in the future
-            pkgs.getopt
-          ]
-          ++ cephDeviceHealthMonitoringPathsOrPackages;
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          preStart = ''
-            ${ensureTransientCephDirs}
-            ${lib.getLib cfg.package}/libexec/ceph/ceph-osd-prestart.sh --cluster ${cfg.clusterName} --id ${toString osdConfig.id}
-          '';
-
-          serviceConfig =
-            let
-              clusterIpArg = lib.optionalString (
-                osdConfig.clusterAddress != null
-              ) "--cluster_addr=${osdConfig.clusterAddress}";
-            in
-            {
-              LimitNOFILE = "1048576";
-              LimitNPROC = "1048576";
-
-              ExecStart = ''
-                ${cfg.package}/bin/ceph-osd -f --cluster ${cfg.clusterName} --id ${toString osdConfig.id} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.osdBindAddr}" "--public_addr=${cfg.osdAdvertisedPublicAddr}" "${clusterIpArg}"
-              '';
-              ExecReload = ''
-                ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-              '';
-              Restart = "on-failure";
-              ProtectHome = "true";
-              ProtectSystem = "full";
-              PrivateTmp = "true";
-              TasksMax = "infinity";
-              # StartLimitBurst="3";
-            };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-    in
-    mkIf cfg.enable {
-      environment.systemPackages = [ cfg.package ];
-
-      networking.firewall = {
-        allowedTCPPorts = [
-          # Ceph outside of VPN because it is very data heavy and causes packet loss.
-          # We enable msgr-v2 only because that allows its own on-wire encryption.
-          3300 # ceph msgr-v2
-        ];
-        allowedTCPPortRanges = [
-          {
-            from = 6800;
-            to = 7300;
-          } # https://docs.ceph.com/en/pacific/rados/configuration/network-config-ref/
-        ];
-      };
-
-      # Reminder of how `ceph.conf` works:
-      #
-      # * Ceph upstream docs now recommend to use underscores instead of spaces.
-      # * Options in more specific sections like `[mon]` override those in less
-      #   specific sections like `[global]`. But all options can be written into all sections,
-      #   and an option has the same name, no matter in which section it is written.
-      #   Thus, put options in `[global]`, and only use a diffent section
-      #   if you want to override an option you've set in `global`.
-      #
-      # Sample: https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf
-      environment.etc."ceph/${cfg.clusterName}.conf".text = ''
-        [global]
-        fsid = ${cfg.fsid}
-        mon_initial_members = ${commaSep (map (mon: mon.hostname) cfg.initialMonitors)}
-        mon_host = ${commaSep (map (mon: mon.ipAddress) cfg.initialMonitors)}
-
-        # Ceph clusters go into WARN health mode, until
-        # the following setting is made strict by setting it to `false`:
-        # See: https://docs.ceph.com/en/latest/security/CVE-2021-20288/#recommendations
-        # As of writing, this setting is not documented outside of the CVE note :(
-        #
-        # While for new clusters the warning no longer seems to appear, it still
-        # appears in our existing clusters unless this option is set, see:
-        # https://tracker.ceph.com/issues/53751#note-7
-        auth_allow_insecure_global_id_reclaim = false
-
-        # Disable dirfrag prefetch on MDS restart to prevent out-of-memory after
-        # many files were opened.
-        # Note this option has no effect on Ceph < 15, because it doesn't exist there.
-        # TODO: Remove this once we're on a Ceph version that includes this default,
-        #       see https://github.com/ceph/ceph/pull/44667.
-        #       This is assuming that the commit fixes existing clusters, see
-        #       https://github.com/ceph/ceph/pull/44667#issuecomment-1036103397
-        #       If it doesn't this can only be removed once we have no existing
-        #       cluster with the old default.
-        mds_oft_prefetch_dirfrags = false
-
-        # Disable sleep between HDD recovery operations, otherwise recovery
-        # will take forever when small objects (e.g. CephFS files) are on HDD.
-        # See https://tracker.ceph.com/issues/23595#note-12
-        osd_recovery_sleep_hdd = 0.0
-
-        # Increase scrub intervals by 4x.
-        # Since we store many small files on HDD, and scrubbing apparently
-        # iterates over all objects
-        # we have no chance to scrub at the default intervals.
-        #
-        # (This was written when we had 400M files across 30 HDDs.)
-        # Change this back once we have reduced our number of files per disk.
-        osd_scrub_min_interval = 345600
-        osd_scrub_max_interval = 2419200
-        osd_deep_scrub_interval = 2419200
-
-        public_network = ${commaSep cfg.publicNetworks}
-        cluster_network = ${commaSep cfg.clusterNetworks}
-        auth_cluster_required = cephx
-        auth_service_required = cephx
-        auth_client_required = cephx
-
-        # Enforce on-wire transport encryption.
-        ms_cluster_mode = secure
-        ms_service_mode = secure
-        ms_client_mode = secure
-
-        ${cfg.extraConfig}
-      '';
-
-      environment.etc."ceph/${cfg.clusterName}.client.admin.keyring" = {
-        source = cfg.adminKeyring;
-        mode = "0600";
-        # Make ceph own this keyring so that it can use it to get keys for its daemons.
-        user = "ceph";
-        group = "ceph";
-      };
-
-      users.users.ceph = {
-        isNormalUser = false;
-        isSystemUser = true;
-        # TODO: Legacy UID / GID chosen from before we configured the UID declaratively.
-        #       In the future, we whould change this whole module to use
-        #       `config.ids.uids.ceph`, like the upstream nixpkgs Ceph module does.
-        #       Switching away from `nogroup` would also be good as described there.
-        #       For both cases, we'll have to `chown` all relevant existing files on
-        #       deployments, such as `/var/lib/ceph`, and log files.
-        uid = 1001;
-        group = config.users.groups.nogroup.name;
-      };
-      users.groups.ceph = {
-        # TODO: Same TODO as above for the `uid`.
-        gid = 499;
-      };
-
-      # Allow ceph daemons (which run as user ceph) to collect device health metrics.
-      security.sudo.extraRules = map (
-        { sudoersExtraRule, ... }: sudoersExtraRule
-      ) cephMonitoringSudoersCommandsAndPackages;
-
-      # The udevadm trigger/settle in `makeCephOsdSetupSystemdService` waits for these rules rule to be applied.
-      services.udev.extraRules = lib.concatStringsSep "\n" (
-        lib.mapAttrsToList (
-          _localOsdServiceName: osdConfig:
-          ''
-            SUBSYSTEM=="block", ${osdConfig.blockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
-          ''
-          + lib.optionalString (osdConfig.dbBlockDeviceUdevRuleMatcher != null) (''
-            SUBSYSTEM=="block", ${osdConfig.dbBlockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
-          '')
-        ) cfg.osds
-      );
-
-      systemd.services = {
-
-        ceph-mon-setup = mkIf cfg.monitor.enable {
-          description = "Initialize ceph monitor";
-
-          preStart = ensureCephDirs;
-
-          script =
-            let
-              # `--addv` seems currently required to get msgr-v2 working, see:
-              #     https://tracker.ceph.com/issues/53751#note-11
-              monmapNodes = builtins.concatStringsSep " " (
-                lib.concatMap (mon: [
-                  "--addv"
-                  mon.hostname
-                  "[v2:${mon.ipAddress}:3300,v1:${mon.ipAddress}:6789]"
-                ]) cfg.initialMonitors
-              );
-            in
-            # Monitors cannot simply be changed in config, one has to update the monmap, see note [replacing-ceph-monmap-ips-for-existing-cluster]
-            ''
-              set -euo pipefail
-              rm -rf "${monDir}" # Start from scratch.
-              echo "Initializing monitor."
-              MONMAP_DIR=`mktemp -d`
-              ${cfg.package}/bin/monmaptool --create ${monmapNodes} --fsid ${cfg.fsid} "$MONMAP_DIR/monmap"
-              ${cfg.package}/bin/ceph-mon --cluster ${cfg.clusterName} --mkfs -i ${cfg.monitor.nodeName} --monmap "$MONMAP_DIR/monmap" --keyring ${cfg.monitor.initialKeyring}
-              rm -r "$MONMAP_DIR"
-              touch ${monDir}/done
-            '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${monDir}/done";
-          };
-        };
-
-        ceph-mon = mkIf cfg.monitor.enable {
-          description = "Ceph monitor";
-
-          requires = [ (ensureUnitExists config "ceph-mon-setup.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-mon-setup.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          path = cephDeviceHealthMonitoringPathsOrPackages;
-
-          preStart = ensureTransientCephDirs;
-
-          serviceConfig = {
-            LimitNOFILE = "1048576";
-            LimitNPROC = "1048576";
-            ExecStart = ''
-              ${cfg.package}/bin/ceph-mon -f --cluster ${cfg.clusterName} --id ${cfg.monitor.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.monitor.bindAddr}" "--public_addr=${cfg.monitor.advertisedPublicAddr}"
-            '';
-            ExecReload = ''
-              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-            '';
-            PrivateDevices = "yes";
-            ProtectHome = "true";
-            ProtectSystem = "full";
-            PrivateTmp = "true";
-            TasksMax = "infinity";
-            Restart = "on-failure";
-            # StartLimitBurst="5";
-            RestartSec = "10";
-          };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-        ceph-mgr-setup = mkIf cfg.manager.enable {
-          description = "Initialize Ceph manager";
-
-          preStart = ensureCephDirs;
-
-          script = ''
-            set -euo pipefail
-            mkdir -p ${mgrDir}
-            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
-            do
-              sleep 1
-            done
-            ${cfg.package}/bin/ceph auth get-or-create mgr.${cfg.manager.nodeName} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o ${mgrDir}/keyring
-            touch "${mgrDir}/.nix_done"
-          '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${mgrDir}/.nix_done";
-          };
-        };
-
-        ceph-mgr = mkIf cfg.manager.enable {
-          description = "Ceph manager";
-
-          requires = [ (ensureUnitExists config "ceph-mgr-setup.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-mgr-setup.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          preStart = ensureTransientCephDirs;
-
-          serviceConfig = {
-            LimitNOFILE = "1048576";
-            LimitNPROC = "1048576";
-
-            ExecStart = ''
-              ${cfg.package}/bin/ceph-mgr -f --cluster ${cfg.clusterName} --id ${cfg.manager.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name}
-            '';
-            ExecReload = ''
-              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-            '';
-            Restart = "on-failure";
-            RestartSec = 10;
-            # StartLimitBurst="3";
-          };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-        ceph-mds-setup = mkIf cfg.mds.enable {
-          description = "Initialize Ceph MDS";
-
-          preStart = ensureCephDirs;
-
-          script = ''
-            set -euo pipefail
-            mkdir -p ${mdsDir}
-            until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
-            do
-              sleep 1
-            done
-            ${cfg.package}/bin/ceph auth get-or-create mds.${cfg.mds.nodeName} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o ${mdsDir}/keyring
-            touch "${mdsDir}/.nix_done"
-          '';
-
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            PermissionsStartOnly = true; # only run the script as ceph
-            User = config.users.users.ceph.name;
-            Group = config.users.groups.ceph.name;
-          };
-          unitConfig = {
-            ConditionPathExists = "!${mdsDir}/.nix_done";
-          };
-        };
-
-        ceph-mds = mkIf cfg.mds.enable {
-          description = "Ceph MDS";
-
-          requires = [ (ensureUnitExists config "ceph-mds-setup.service") ];
-          requiredBy = [ "multi-user.target" ];
-          after = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-            (ensureUnitExists config "ceph-mds-setup.service")
-          ];
-          wants = [
-            "network.target"
-            "local-fs.target"
-            "time-sync.target"
-          ];
-
-          restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
-          preStart = ensureTransientCephDirs;
-
-          serviceConfig = {
-            LimitNOFILE = "1048576";
-            LimitNPROC = "1048576";
-
-            ExecStart = ''
-              ${cfg.package}/bin/ceph-mds -f --cluster ${cfg.clusterName} --id ${cfg.mds.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_addr=${cfg.mds.listenAddr}"
-            '';
-            ExecReload = ''
-              ${pkgs.coreutils}/bin/kill -HUP $MAINPID
-            '';
-            Restart = "on-failure";
-            # StartLimitBurst="3";
-          };
-          # startLimitIntervalSec = 30 * 60;
-        };
-
-      }
-      # Make one OSD service for each configured OSD.
-      // lib.mapAttrs' (
-        localOsdServiceName: osdConfig:
-        nameValuePair "ceph-osd-setup-${localOsdServiceName}" (
-          makeCephOsdSetupSystemdService localOsdServiceName osdConfig
-        )
-      ) cfg.osds
-      // lib.mapAttrs' (
-        localOsdServiceName: osdConfig:
-        nameValuePair "ceph-osd-${localOsdServiceName}" (
-          makeCephOsdSystemdService localOsdServiceName osdConfig
-        )
-      ) cfg.osds;
-    };
-}

modules/nixos/default.nix

@@ -7,7 +7,6 @@
     ./adblock-update.nix
     ./albyhub.nix
     ./backup.nix
-    ./ceph.nix
     ./db.nix
     ./gitea-runner.nix
     ./gnome.nix
@@ -16,6 +15,7 @@
     #./kiwix-serve.nix
     ./kubernetes.nix
     ./linode.nix
+    ./monitoring-access.nix
     ./podman.nix
     ./print.nix
     ./proxy.nix

modules/nixos/gitea-runner.nix

@@ -98,6 +98,19 @@ in
         inherit labels;
         inherit (cfg) name;
         enable = true;
+        hostPackages = with pkgs; [
+          bash
+          buildah
+          coreutils
+          curl
+          gawk
+          gitMinimal
+          gnused
+          nix
+          nodejs
+          podman
+          wget
+        ];
         url = cfg.instanceURL;
         tokenFile = config.age.secrets."gitea-runner-${host}-podman".path;
         settings = {

modules/nixos/home.nix

@@ -1,7 +1,6 @@
 {
   config,
   lib,
-  pkgs,
   ...
 }:
 
@@ -18,35 +17,9 @@ with lib;
   };
 
   config = mkIf cfg {
-    age.secrets.attic.file = ../../secrets/attic.age;
     networking.domain = "thehellings.lan";
     time.timeZone = "America/Chicago";
 
-    systemd.services.attic-client = {
-      enable = true;
-      description = "Attic client watch-store service";
-      after = [ "network.target" ];
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        Type = "simple";
-        Restart = "on-failure";
-        RestartSec = "5s";
-      };
-      preStart = ''
-        set -x
-        mkdir -p $XDG_CONFIG_HOME/attic
-        cp ${config.age.secrets.attic.path} $XDG_CONFIG_HOME/attic/config.toml
-      '';
-      script = "${pkgs.attic-client}/bin/attic watch-store --ignore-upstream-cache-filter default";
-      environment = {
-        XDG_CONFIG_HOME = "/var/lib/attic-client";
-      };
-    };
-
-    systemd.tmpfiles.rules = [
-      "d /var/lib/attic-client 0755 root root -"
-    ];
-
     # Open Prometheus exporter ports on LAN-connected hosts only.
     # NOT in baseline.nix to avoid exposing these on internet-facing hosts (e.g. linode).
     networking.firewall.allowedTCPPorts = [

modules/nixos/kubernetes.nix

@@ -112,7 +112,8 @@ in
       keepalived = {
         enable = true;
         openFirewall = true;
-        vrrpInstances.kubernetes = {
+        vrrpInstances = {
+          kubernetes = {
             interface = cfg.vipInterface;
             priority = cfg.priority;
             state = if (config.networking.hostName == "isaiah") then "MASTER" else "BACKUP";
@@ -128,6 +129,7 @@ in
             '';
           };
         };
+      };
       openiscsi = {
         enable = true;
         name = "${config.networking.hostName}-initiatorhost";

modules/nixos/monitoring-access.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.greg.monitoring-access;
+in
+with lib;
+{
+  options.greg.monitoring-access = {
+    enable = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Create a dedicated, read-only account (`emily`) for automated
+        monitoring and analysis by the Hermes agent. The account is
+        SSH-key-only (no password set), is not added to `wheel`, and is
+        granted no sudo rights. It only gets read access to the systemd
+        journal via group membership, which is sufficient for log
+        inspection and health/analysis tasks without any privileged
+        access to the rest of the system.
+      '';
+    };
+
+    sshKeys = mkOption {
+      type = types.listOf types.str;
+      default = lib.strings.splitString "\n" (
+        builtins.readFile ../../home/ssh/emily_authorized_keys
+      );
+      description = "SSH public keys authorized to log in as the monitoring account.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups.emily = { };
+
+    users.users.emily = {
+      isNormalUser = true;
+      createHome = true;
+      description = "Read-only monitoring/analysis account (Hermes agent)";
+      group = "emily";
+      # No password is set on purpose: this account is SSH-key-only.
+      extraGroups = [
+        "systemd-journal" # read access to the journal for log analysis
+      ];
+      shell = pkgs.bashInteractive;
+      openssh.authorizedKeys.keys = cfg.sshKeys;
+    };
+  };
+}

modules/nixos/vmdev.nix

@@ -23,6 +23,13 @@ with lib;
         type = types.str;
         description = "Kernel module type to install - amd, intel, etc";
       };
+      host = mkOption {
+        type = types.enum [
+          "libvirt"
+          "vbox"
+        ];
+        description = "Which VM hosting type to configure";
+      };
     };
   };
 
@@ -35,7 +42,6 @@ with lib;
       nixos-generators
       packer
       swtpm
-      virt-manager
       virtio-win
       xorriso
     ];
@@ -44,7 +50,7 @@ with lib;
 
     # Enable the virtualisation services
     virtualisation = {
-      libvirtd = {
+      libvirtd = mkIf (cfg.host == "libvirt") {
         enable = true;
         onBoot = "ignore"; # Do not auto-restart VMs on boot, unless they are marked autostart
         qemu = {
@@ -54,6 +60,11 @@ with lib;
           };
         };
       };
+      virtualbox.host = mkIf (cfg.host == "vbox") {
+        enable = true;
+        enableExtensionPack = true;
+        headless = true;
+      };
     };
 
     boot.extraModprobeConfig = "options kvm_${cfg.system} nested=1";

network.json

@@ -9,13 +9,6 @@
     "tailscale": "100.64.0.0/10"
   },
   "hosts": {
-    "chronicles": {
-      "ip": "10.42.1.4",
-      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEBecZUva9OnZuXLaBun6/1ITo5f9p0YMLPD+q0egLRS",
-      "external": true,
-      "ts": "100.119.228.115",
-      "aliases": ["s3"]
-    },
     "exodus": {
       "ip": null,
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3",
@@ -31,6 +24,15 @@
       "tags": ["router", "server"],
       "nebulaIp": "10.157.0.2"
     },
+    "gregory.hellings-mbp": {
+      "external": true,
+      "system": "aarch64-darwin",
+      "user": "gregory.hellings"
+    },
+    "gregs-MacBook-Pro-16-inch-Nov-2024": {
+      "external": true,
+      "system": "aarch64-darwin"
+    },
     "hosea": {
       "ip": "10.42.1.7",
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKLIwkTTXA56sUlUjEulXXZRvZy5H4a5ZwgKWLlpkQDz",
@@ -39,10 +41,6 @@
       "tags": ["server"],
       "nebulaIp": "10.157.0.3"
     },
-    "icdm-root": {
-      "external": true,
-      "system": "x86_64-linux"
-    },
     "isaiah": {
       "builder": true,
       "ip": "10.42.1.6",
@@ -57,12 +55,12 @@
       "external": true,
       "system": "x86_64-linux"
     },
-    "gregory.hellings-mbp": {
+    "ivr": {
       "external": true,
-      "system": "aarch64-darwin",
+      "system": "aarch64-darwin"
-      "user": "gregory.hellings"
     },
     "jeremiah": {
+      "aliases": ["buildbot"],
       "builder": true,
       "ip": "10.42.1.8",
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOjQjXq9WYU2Ki27BR9WwJ4ZruS/lJXbjC1b0Q42Adi0",
@@ -72,20 +70,19 @@
       "tags": ["builder", "kube", "server"],
       "nebulaIp": "10.157.0.5"
     },
-    "joel": {
+    "kuma": {
-      "external": true,
+      "ip": "10.42.1.19",
-      "ip": "10.42.0.4",
+      "nebulaIp": "10.157.0.8",
-      "ts": null
+      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIhr+LmYMOk4Hixxew2FiAvL8sycgQvnhK8PBGjfnkJb",
-    },
+      "system": "x86_64-linux",
-    "gregs-MacBook-Pro-16-inch-Nov-2024": {
+      "tags": ["server"]
-      "external": true,
-      "system": "aarch64-darwin"
     },
     "lithic": {
       "external": true,
       "system": "aarch64-darwin"
     },
     "linode": {
+      "connectAddr": "thehellings.com",
       "ip": null,
       "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMv9Zud3kZOl86gtmkn+uj3D4kiXWDPtyUL02VVLNR4Q",
       "ts": "100.109.86.8",
@@ -93,10 +90,6 @@
       "tags": ["public", "server"],
       "nebulaIp": "10.157.0.1"
     },
-    "ivr": {
-      "external": true,
-      "system": "aarch64-darwin"
-    },
     "MacBook-Pro.local": {
       "external": true,
       "system": "aarch64-darwin"
@@ -105,38 +98,10 @@
       "external": true,
       "system": "aarch64-darwin"
     },
-    "nas1": {
-      "external": true,
-      "ip": "10.42.1.14",
-      "ts": "100.114.187.61"
-    },
     "nixos": {
       "external": true,
       "system": "x86_64-linux"
     },
-    "printer": {
-      "external": true,
-      "ip": "10.42.1.3"
-    },
-    "proxmoxtemplate": {
-      "external": true,
-      "system": "x86_64-linux"
-    },
-    "pve2": {
-      "external": true,
-      "ip": "10.42.1.15",
-      "system": "x86_64-linux"
-    },
-    "pve3": {
-      "external": true,
-      "ip": "10.42.1.16",
-      "system": "x86_64-linux"
-    },
-    "pve4": {
-      "external": true,
-      "ip": "10.42.1.17",
-      "system": "x86_64-linux"
-    },
     "wsl": {
       "external": true,
       "system": "aarch64-linux"
@@ -151,5 +116,57 @@
       "tags": ["builder", "kube", "server"],
       "nebulaIp": "10.157.0.6"
     }
+  },
+  "external": {
+    "chronicles": {
+      "ip": "10.42.1.4",
+      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEBecZUva9OnZuXLaBun6/1ITo5f9p0YMLPD+q0egLRS",
+      "ts": "100.119.228.115",
+      "aliases": ["s3"]
+    },
+    "hermes": {
+      "ip": "10.42.1.18",
+      "mac": "BC:24:11:E4:72:AB",
+      "nebulaIp": "10.157.0.8",
+      "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILFYyzz/9i5rXprCQj9IL1ulrbQ6E9BOSeOcvf4D/b0G",
+      "tags": ["server"]
+    },
+    "k3s": {
+      "aliases": ["*.k3s"],
+      "ip": "10.42.5.1",
+      "nebulaIp": "10.157.100.1"
+    },
+    "nas1": {
+      "aliases": ["*.nas1"],
+      "ip": "10.42.1.14",
+      "ts": "100.114.187.61"
+    },
+    "printer": {
+      "ip": "10.42.1.3"
+    },
+    "pve1": {
+      "ip": "10.42.0.4"
+    },
+    "pve2": {
+      "ip": "10.42.1.15"
+    },
+    "pve2bmc": {
+      "ip": "10.42.6.2",
+      "mac": "00:25:90:4b:34:e8"
+    },
+    "pve3": {
+      "ip": "10.42.1.16"
+    },
+    "pve3bmc": {
+      "ip": "10.42.6.3",
+      "mac": "00:25:90:4a:dc:2e"
+    },
+    "pve4": {
+      "ip": "10.42.1.17"
+    },
+    "pve4bmc": {
+      "ip": "10.42.6.4",
+      "mac": "00:25:90:4a:d8:2e"
+    }
   }
 }

pkgs/default.nix

@@ -16,9 +16,12 @@ let
     adblock_update = c ./adblock_update.nix { };
     brew = c ./homebrew.nix { };
     create_ssl = c ./create_ssl.nix { };
-    dockerCompat = pkgs.runCommand "docker-compat" {
+    dockerCompat =
-      nativeBuildInputs  = [];
+      pkgs.runCommand "docker-compat"
-    } ''
+        {
+          nativeBuildInputs = [ ];
+        }
+        ''
           mkdir -p $out/bin
           ln -s ${pkgs.podman}/bin/podman $out/bin/docker
         '';
@@ -29,6 +32,7 @@ let
     inject = c ./inject.nix { };
     setup-ssh = c ./setup-ssh { };
     upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
+    zim-updater = c ./zim/updater.nix { };
   };
   x86Linux = {
     qemu-hook = c ./qemu-hook.nix { };

pkgs/zim/blobs.json

@@ -2,18 +2,18 @@
   "en": {
     "gutenberg": {
       "name": "gutenberg_en_all",
-      "version": "2023-08",
+      "version": "2025-11",
-      "hash": "sha256-OXmdHdsLZcW4nCUQsy7sMpUssS8NV7ztkCgS/nsISuw="
+      "hash": "sha256-AWd8jVVKHKssv9AgrJnryn3Wx0084NE/JmorpgPfryg="
     },
     "phet": {
       "name": "phet_en_all",
-      "version": "2025-03",
+      "version": "2026-05",
-      "hash": "sha256-ARuUzU2o17J2/ZedHc2acXl33dtHCbQJEb72UQUEm1Y="
+      "hash": "sha256-zAAq/X5rjQUiYmjMpBtWP5z3J2ZHJMmIxFKnxLAhOlA="
     },
     "wikibooks": {
       "name": "wikibooks_en_all_maxi",
-      "version": "2025-10",
+      "version": "2026-04",
-      "hash": "sha256-ONBh/ze1Fv2Ffm5b9vDzYS/i2cWSa4pM4MLZnozr2n8="
+      "hash": "sha256-wt7Zr+RkfCsFrOudMCYBADacu6IvPQDkZ6Y0VG2j9hA="
     },
     "wikipedia": {
       "name": "wikipedia_en_all_maxi",
@@ -22,40 +22,40 @@
     },
     "wikisource": {
       "name": "wikisource_en_all_maxi",
-      "version": "2025-11",
+      "version": "2026-05",
-      "hash": "sha256-p1Jio+PMTZVSLIDIOBeMG3acXJg83CwQM7zIWDUyozM="
+      "hash": "sha256-OA4b+U8mxpcX3fst6hrMyctucYTQOlG/b4qZDiVlcf4="
     },
     "wikiversity": {
       "name": "wikiversity_en_all_maxi",
-      "version": "2025-11",
+      "version": "2026-05",
-      "hash": "sha256-IG/gAUdc/vHRrIWGHhw8vMJdLWwqrNbfh+SiclQRIAI="
+      "hash": "sha256-8mZ1CSUcF4QnDIyN0M2KedQ4pcSUCmzHBlOm93MYpCE="
     },
     "wiktionary": {
       "name": "wiktionary_en_all_nopic",
-      "version": "2025-09",
+      "version": "2026-05",
-      "hash": "sha256-Ghcb60qeGaSJtTKQkJoMx/XucX7Lt1XY145lD3gHlMg="
+      "hash": "sha256-Dwiz+viVQt0zb077R9KQRgtUtxG9ixA/DeXAkCdD4rM="
     }
   },
   "fr": {
     "gutenberg": {
       "name": "gutenberg_fr_all",
-      "version": "2025-10",
+      "version": "2026-01",
-      "hash": "sha256-gLHxLXJNwwcxM/mZHtoJ8ojbGQ1fugxgni/+RXccwhU="
+      "hash": "sha256-sGn1TeKwBK0+Er5YOJWq6g0itEm4gIGvKxv/PW0DIao="
     },
     "phet": {
       "name": "phet_fr_all",
-      "version": "2025-03",
+      "version": "2026-05",
-      "hash": "sha256-CSboZNTIowLLhp1u9wkxH8xmu3LvQNx5q493AOYJRIc="
+      "hash": "sha256-bJmchFnaPcofE/hy3ZcRggMT9osHqXwLbwMLJ+cpF28="
     },
     "wikibooks": {
       "name": "wikibooks_fr_all_maxi",
-      "version": "2025-09",
+      "version": "2026-07",
-      "hash": "sha256-KHCc/73L5Bd9iZ47SRV6vpI8fVM1v9hk1TpEvTro2uo="
+      "hash": "sha256-csI+E5UFGi42BLGWWHxQhVr5KSihCFc53KGKwo557Ck="
     },
     "wikipedia": {
       "name": "wikipedia_fr_all_maxi",
-      "version": "2025-06",
+      "version": "2026-05",
-      "hash": "sha256-ve7Mbh96/ObNbV/KVNYZpfLN+HdWlJ7C5LItEdVkRH0="
+      "hash": "sha256-YUAbGzYYr+c2RQqGjHIF9XL1kNsxDgRKnHK1H7/PtDE="
     },
     "wikisource": {
       "name": "wikisource_fr_all_maxi",
@@ -64,25 +64,25 @@
     },
     "wikiversity": {
       "name": "wikiversity_fr_all_maxi",
-      "version": "2025-09",
+      "version": "2026-05",
-      "hash": "sha256-55fdtw/cRezq6nkRp6wuwkukBiqsQKdwh9QWVAgFcBI="
+      "hash": "sha256-nVDHtkWIOJkoiGixnA1zVR1QS58thjc12FJyagK3Ec0="
     },
     "wiktionary": {
       "name": "wiktionary_fr_all_nopic",
-      "version": "2025-11",
+      "version": "2026-05",
-      "hash": "sha256-P0CJptee5rDzccxyRDBzyRLuuFOcaLsJYi6DkBFUfIc="
+      "hash": "sha256-7UXByW2IIL0hec8RPT39jpg5IEGvRMD47hc4Y4vFkJs="
     }
   },
   "ht": {
     "phet": {
       "name": "phet_ht_all",
-      "version": "2025-03",
+      "version": "2026-05",
-      "hash": "sha256-bHrPzE8H7ptrP6r5wPlXSsXNlTiKxJ9KatABC4kwx+s="
+      "hash": "sha256-GiJ1jllhioyMYidQ7+Lcbdd9JN2yf04ZQgnO8XaGAqY="
     },
     "wikipedia": {
       "name": "wikipedia_ht_all_maxi",
-      "version": "2026-04",
+      "version": "2026-07",
-      "hash": "sha256-qUX0pKxIyNsWX4V6kNIsP4Z9nc6TDhwdDR0MmpDOQFs="
+      "hash": "sha256-InS9cMRaIv9ArlKVwKUypz56m4DgSR7Tl6XpSTMzH+o="
     }
   }
 }

pkgs/zim/updater.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strings"
@@ -17,27 +17,144 @@ import (
 
 const BASE = "https://download.kiwix.org/zim"
 
-func getTypes() []string {
+// ///////////////////////////////////////////////////////////////////////////
-	return []string{
+// ////////////////  THE BLOB ITSELF /////////////////////////////////////////
-		"phet",
+// ///////////////////////////////////////////////////////////////////////////
-		"wikipedia",
+type Blobs struct {
-		"wiktionary",
+	En    Language `json:"en"`
-		"wikiversity",
+	Fr    Language `json:"fr"`
-		"wikisource",
+	Ht    Language `json:"ht"`
-		"wikibooks",
+	dirty bool
-		"gutenberg",
+}
-		"ted",
+
+func (b *Blobs) Populate() {
+	done := make(chan bool)
+	waitFor := 0
+	// Launch self-populating efforts
+	blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
+	for f := range blobType.Fields() {
+		if f.IsExported() {
+			//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
+			waitFor += 1
+			go reflect.Indirect(reflect.ValueOf(b)).
+				FieldByName(f.Name).
+				Addr().
+				Interface().
+				(*Language).
+				Populate(strings.ToLower(f.Name), done)
+		}
+	}
+	// Wait until all languages are completed
+	for d := range done {
+		waitFor -= 1
+		if waitFor == 0 {
+			fmt.Println("Completed waiting for all languages")
+			close(done)
+			break
+		}
+		b.dirty = b.dirty || d
 	}
 }
 
-func getLanguages() []string {
+/////////////////////////////////////////////////////////////////////////////
-	return []string{
+//////////////////////// The Language ///////////////////////////////////////
-		"ht",
+/////////////////////////////////////////////////////////////////////////////
-		"en",
+
-		"fr",
+type Language struct {
+	Gutenberg   *Zim `json:"gutenberg,omitempty"`
+	Phet        *Zim `json:"phet,omitempty"`
+	Ted         *Zim `json:"ted,omitempty"`
+	Wikibooks   *Zim `json:"wikibooks,omitempty"`
+	Wikipedia   *Zim `json:"wikipedia,omitempty"`
+	Wikisource  *Zim `json:"wikisource,omitempty"`
+	Wikiversity *Zim `json:"wikiversity,omitempty"`
+	Wiktionary  *Zim `json:"wiktionary,omitempty"`
+	dirty       bool
+}
+
+func (l *Language) Populate(code string, done chan bool) {
+	childDone := make(chan bool)
+	waitFor := 0
+	languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
+	l.dirty = false
+	for f := range languageType.Fields() {
+		if f.IsExported() {
+			ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
+			    FieldByName(f.Name)
+			if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
+				waitFor += 1
+				go reflect.Indirect(ptrPtrZim).
+				    Addr().
+				    Interface().
+				    (*Zim).
+				    Populate(strings.ToLower(f.Name), code, childDone)
+			} else {
+				// TODO: Figure out how to set a value over the nil pointer
+				// of the field, so we can auto-detect when these are available
+				// in the future
+				// waitFor += 1
+				//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
+				//ptrPtrZim.Set(reflect.ValueOf(z))
+				//go z.Populate(strings.ToLower(f.Name), code, childDone)
+			}
+		}
+	}
+	// Wait until children are all done
+	for d := range childDone {
+		waitFor -= 1
+		if waitFor == 0 {
+			close(childDone)
+			break
+		}
+		l.dirty = l.dirty || d
+	}
+
+	//fmt.Printf("%s\tFinished populate\n", code)
+	done <- l.dirty
+}
+
+// ///////////////////////////////////////////////////////////////////////////
+// ////////////////////// A Single Zim ///////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////////////
+type Zim struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Hash    string `json:"hash"`
+	dirty   bool
+}
+
+func (z *Zim) Populate(category string, language string, done chan bool) {
+	//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
+	// Look for a possible Zim file
+	links := getLinks(getPage(category))
+	link, err := getName(links, category, language)
+	if err != nil {
+		fmt.Printf("%s:%s\tNo zim found\n", language, category)
+		done <- false
+		return
+	}
+	//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
+	// Extract name and version
+	re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
+	match := re.FindStringSubmatch(link)
+	if len(match) != 3 {
+		fmt.Printf("%s:%s Matches: %s", language, category, match)
+		os.Exit(1)
+	}
+	// Check if the name and version mismatch
+	if match[1] == z.Name && match[2] == z.Version {
+		fmt.Printf("Skipping existing hash: %s\n", link)
+		done <- false
+	} else {
+		z.Name = match[1]
+		z.Version = match[2]
+		// Fetch the Zim file, if it differs from what we currently have
+		z.UpdateHash(category)
+		done <- true
 	}
 }
 
+// Helper function to fetch the HTML of a given type page
 func getPage(t string) string {
 	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
 	if err != nil {
@@ -48,6 +165,7 @@ func getPage(t string) string {
 	return string(pageBytes)
 }
 
+// Helper function to parse out all the links from the page
 func getLinks(page string) []string {
 	ret := []string{}
 
@@ -60,6 +178,7 @@ func getLinks(page string) []string {
 	return ret
 }
 
+// Helper function to get the most likely link for this particular entry
 func getName(links []string, t, lang string) (string, error) {
 	candidates := []string{}
 	prefix := fmt.Sprintf("%s_%s_all", t, lang)
@@ -79,44 +198,39 @@ func getName(links []string, t, lang string) (string, error) {
 	}
 }
 
-func getHash(ch chan result, file, category, language string) {
+// Helper function to get the hash value from the resulting link
-	// TODO: Only call this if the file doesn't already have a hash
+func (z *Zim) UpdateHash(category string) error {
-	// in the existing file
+	file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)
-	fmt.Printf("Fetching hash for %s\n", file)
+
-	cmd := exec.Command( "nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
+	// First fetch the file into our local Nix store
+	fmt.Printf("Fetching hash: %s\n", file)
+	cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
 	out, err := cmd.Output()
 	if err != nil {
-		fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
+		fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
 		if exitErr, ok := err.(*exec.ExitError); ok {
 			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
 		}
-		ch <- result{category, language, ""}
+		return errors.New("Error fetching file")
-		return
 	}
 	hash := strings.TrimSpace(string(out))
-	fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
+	fmt.Printf("%s: Successfully fetched raw hash\n", file)
-	ch <- result{category, language, hash}
+	fmt.Printf("%s: Hash is: %s\n", file, hash)
-}
 
-func outputIsValid(o map[string]map[string]Zim) bool {
+	// Then, convert the hash to SRI
-	for a := range o {
+	cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
-		for b := range o[a] {
+	out2, err2 := cmd2.Output()
-			if o[a][b].Hash == "" {
+	if err2 != nil {
-				return false
+		fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
 		}
+		return errors.New("Error fetching file")
 	}
-	}
+	z.Hash = strings.TrimSpace(string(out2))
-	return true
+	z.dirty = true
-}
+	fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
-
+	return nil
-type result struct {
-	category, language, hash string
-}
-
-type Zim struct {
-	Name string `json:"name"`
-	Version string `json:"version"`
-	Hash string `json:"hash"`
 }
 
 func main() {
@@ -125,90 +239,31 @@ func main() {
 
 	// Determine the output file path
 	var outputPath string
-	if *outputFile != "" {
 	outputPath = *outputFile
-	} else {
-		// Get the directory where updater.go is located
-		execPath, err := os.Executable()
-		if err != nil {
-			// Fallback to current directory if we can't determine executable path
-			outputPath = "blobs.json"
-		} else {
-			dir := filepath.Dir(execPath)
-			outputPath = filepath.Join(dir, "blobs.json")
-		}
-	}
 
 	fmt.Println("Writing file to ", outputPath)
 
 	// Read existing cache if it exists
-	cached := make(map[string]map[string]Zim)
+	var blobs Blobs
 	if data, err := os.ReadFile(outputPath); err == nil {
-		if err := json.Unmarshal(data, &cached); err != nil {
+		if err := json.Unmarshal(data, &blobs); err != nil {
 			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
 		} else {
 			fmt.Printf("Loaded existing cache from %s\n", outputPath)
 		}
+	} else {
+		fmt.Printf("Error reading file: %s", err)
+		os.Exit(1)
 	}
+	blobs.Populate()
 
-	output := make(map[string]map[string]Zim)
+	ret, err := json.MarshalIndent(&blobs, "", "  ")
-	comms := make(chan result)
-	pendingHashes := 0
-
-	for _, t := range getTypes() {
-		page := getPage(t)
-		links := getLinks(page)
-		for _, lang := range getLanguages() {
-			if file, err := getName(links, t, lang); err == nil {
-				if _, ok := output[lang]; !ok {
-					output[lang] = make(map[string]Zim)
-				}
-
-				// Check if this file already exists in cache with same name
-				if cachedLang, ok := cached[lang]; ok {
-					if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
-						// Reuse cached hash
-						fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
-						output[lang][t] = cachedEntry
-						continue
-					}
-				}
-
-				// File is new or name has changed, fetch hash
-				output[lang][t] = Zim{file, ""}
-				pendingHashes++
-				go getHash(comms, file, t, lang)
-			}
-		}
-	}
-
-	// Only wait for results if we actually spawned goroutines
-	if pendingHashes > 0 {
-		hashesReceived := 0
-		for r := range comms {
-			if entry, ok := output[r.language][r.category]; ok {
-				entry.Hash = r.hash
-				output[r.language][r.category] = entry
-			}
-			hashesReceived++
-			if hashesReceived >= pendingHashes {
-				close(comms)
-				break
-			}
-		}
-	}
-
-	// Verify all hashes are present
-	if !outputIsValid(output) {
-		fmt.Println("Warning: Some hashes are missing from the output")
-	}
-	ret, err := json.MarshalIndent(output, "", "  ")
 	if err != nil {
 		fmt.Printf("Error marshaling JSON: %v\n", err)
 		os.Exit(1)
 	}
 
-	err = os.WriteFile(outputPath, ret, 0644)
+	err = os.WriteFile(outputPath, []byte(fmt.Sprintf("%s\n", ret)), 0644)
 	if err != nil {
 		fmt.Printf("Error writing to file %s: %v\n", outputPath, err)
 		os.Exit(1)

secrets/acme_password.age

@@ -1,37 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 87huqg LKhlbZI1BygDI8E4kkzhw0/Y1BfWnzekf/URa/Bma30
-uT7QzE8DPX8j3us7oNRKwu6hPUFpQosbcMRmMNgS/Gw
--> ssh-ed25519 8UnW5Q dDAJI5EnPMIhs5H21wLT84GWhkROEJRXE1QVp1aOH38
-qZJt4Kip5qVZl9QoIFEXgrhtrlQEpbkfLarCgaHugj8
--> ssh-ed25519 UFfTmg kbrSoge204JvRPBszhAfMewPM4oHg0HkF52BNeb9kx8
-E4ASRdn0pjF//GiFCjmRlgXRS2wiJxIXDFSjldMtTdQ
--> ssh-ed25519 xNtnoA 7b9/Z/qUEmRnFCiCYzVGJVvxvV8onjZemSg6sllaDyA
-bQosZIGbaSsUqFO2JuRpIhm3FJd79JyYS43A9+zr+1w
--> ssh-ed25519 aY2AXA m4gIzzAX8pkBHHzsSykIy24E6Kru/I4uNnjihKH3JHg
-S18CWIY5wNcR5PVkCqbjp1FJRrGxco6fdoykQeHhyX0
--> ssh-ed25519 AQhf1g LBmvyUbKRLKtPGIEqt+NwOSrtGwlFt2zqgdmRDHu9n8
-zb8OnKRsJBEkH6eJ76mlKvJmtZ8NvVwVS9/0tsZlMUk
--> ssh-ed25519 mOmPfg TuPbRMlX9eRATxSbv6v88TtzGhOKWn5FOkIg4xR6FH0
-NLNzJM/9r2jZ70Y2r5Hsc36gcRTSPw40Aabn/H7OE+4
--> ssh-ed25519 YJiRbw bVMyOOgBcFFRNulym46YwKAIPlOkbQhmQAo9kl4dRx8
-5W2fIxGXCXbzULOismGg4NxE+VQ8LDEbODlJ0OpFMiI
--> ssh-ed25519 Nl/5yA ydFS0bVjjDzymhQFaBJ3YqDREQDpVO6VzgS2MBtuOUo
-tNzj/85BiYe6th2N6A9ZkMSPKduwAIe/qnf2NVJiHOg
--> ssh-ed25519 GdLgCQ wmfT7bCzwnkwEtn2mRnElIM75MmdRw+33MhSegSHJWk
-YHrCYB1wF1njNjzpIy3hKZ9l1cj7m5yicLuj71TvKr8
--> ssh-ed25519 tOH/HQ z5/GMQOtkpMWwWdMXqxmf12MM84xkgXL/OLliq3RHh0
-mVhBt+uc7z/YkoIAyXTqBgv00cJOrZ0bsBP0lD9S24c
--> ssh-ed25519 FpzvfQ D8N5JQ+3rGdSZIPmzlZqO5VpS90v6r99WMXtyUqCGiw
-A2SSvI4H5TRY13R/iOa60bCdc0zR6lUAHJVHfAUR5mQ
--> ssh-ed25519 2UotMw eaM1Kkzw53Oq8lW0BFn3rqWFcTQaTz9jcDsULDg/fwU
-3QVYFrhKnvwWDpIdsyTp0amontUXePko8z8PeUub9EE
--> ssh-ed25519 kdPvzQ 08LE8a24S5cvuK9DVWD2ta0GjJMNzUidmgYT4RfhQDA
-cRFH8vOr2y1/C9F2wZWV2deUxUpoL34IFRyOBHPYifI
--> ssh-ed25519 onmXpg J5MKGIovNwv4FyTN4ofExlHV8qYzU/J9O3MI4tZYcTo
-iagT0Ypo8gTwvGpV66XNuNPedefx2S72q9hyeB2Rz/w
--> ssh-ed25519 CnhD0g se/N4hrYxOiukByYzsIhxToceXNuBO7J7VmM4muJ3Dk
-cW+WW8iWjj3eXDqDO8aEON1ZH/+6AXQGAEXkLRbL3AI
--> ssh-ed25519 4ep2UA uyjvubY39oSGeCNsX3/VIdM5CzNBlrnRpkF+YN5MBCo
-xiNuksI7olV5Db392Vjz/7uDNDIssGmI9bez6vSOXTk
---- VsGbd8K7JzHe/eoYUeTMwcg/GrY9SXVzNxL4HZ23tKQ
-£_÷t/¢ªØËVÎÞ}7J'%?™Á‰Å0“¤¼^õ¿-„ÏÛ„äu4¥ù‹íäóü0!®ÝîËðMÜÅ·