hosts/unstable/hosea/default.nix

@@ -26,6 +26,13 @@ in
     owner = "grafana";
   };
 
+  # NOTE: Before deploying, create the secret with:
+  #   agenix -e secrets/albyhub-restic-password.age
+  age.secrets.albyhub-restic-password = {
+    file = ../../../secrets/albyhub-restic-password.age;
+    owner = "root";
+  };
+
   # Bootloader
   boot = {
     loader = {
@@ -103,6 +110,29 @@ in
         workDir = "/chain/alby";
       };
     };
+    restic.backups.albyhub = {
+      # Backs up AlbyHub's LDK state directory to the nas1 Restic REST server.
+      # The service is stopped before backup to ensure LDK state consistency
+      # and restarted afterward.
+      repository = "rest:https://nas1.shire-zebra.ts.net:30248/albyhub";
+      passwordFile = config.age.secrets.albyhub-restic-password.path;
+      paths = [ "/chain/alby" ];
+      timerConfig = {
+        OnCalendar = "daily";
+        RandomizedDelaySec = "1h";
+      };
+      backupPrepareCommand = ''
+        systemctl stop albyhub.service || true
+      '';
+      backupCleanupCommand = ''
+        systemctl start albyhub.service || true
+      '';
+      pruneOpts = [
+        "--keep-daily 7"
+        "--keep-weekly 4"
+        "--keep-monthly 3"
+      ];
+    };
     jellyfin = {
       enable = true;
       openFirewall = true;

manifests/gitea/dump-cronjob.yaml

@@ -0,0 +1,44 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-dump
+  namespace: gitea
+spec:
+  schedule: "0 2 * * *" # At 2am each day (offset from 4am Immich backup)
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            fsGroup: 1000
+          volumes:
+            - name: gitea-data
+              persistentVolumeClaim:
+                claimName: gitea-release-gitea
+                readOnly: true
+            - name: nas1-dumps
+              nfs:
+                server: nas1.shire-zebra.ts.net
+                path: /mnt/all/backups/gitea-dumps
+          containers:
+            - name: gitea-dump
+              image: "gitea/gitea:1.25.4"
+              command:
+                - /bin/sh
+                - "-c"
+                - |
+                  set -e
+                  DUMP_FILE="/nas1/gitea-dump-$(date +%Y%m%d-%H%M%S).zip"
+                  gitea dump -c /data/gitea/conf/app.ini --type zip --file "$DUMP_FILE"
+                  echo "Dump written to $DUMP_FILE"
+                  find /nas1/ -name "*.zip" -mtime +30 -delete
+                  echo "Old dumps cleaned up"
+              volumeMounts:
+                - name: gitea-data
+                  mountPath: /data
+                  readOnly: true
+                - name: nas1-dumps
+                  mountPath: /nas1

manifests/gitea/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - chart.yaml
   - ingress.yaml
   - secrets.yaml
+  - dump-cronjob.yaml

manifests/immich/backup.yaml

@@ -19,6 +19,32 @@ spec:
         key: "dd0de40c-9ad6-4034-9d5c-b311016ebf19"
         property: apikey
 ---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: immich-minio-creds
+spec:
+  target:
+    name: immich-minio-creds
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        minio_key: "{{ .minio_key }}"
+        minio_secret: "{{ .minio_secret }}"
+  secretStoreRef:
+    name: bitwarden-fields
+    kind: ClusterSecretStore
+  data:
+    - secretKey: minio_key
+      remoteRef:
+        key: "dcbcf704-7dce-48d7-bbd1-b3a801875b3d"
+        property: username
+    - secretKey: minio_secret
+      remoteRef:
+        key: "dcbcf704-7dce-48d7-bbd1-b3a801875b3d"
+        property: password
+---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -31,11 +57,9 @@ spec:
         spec:
           restartPolicy: Never
           volumes:
-            - name: nas1
-              nfs:
-                path: /mnt/all/backups/immich
-                server: nas1.shire-zebra.ts.net
-          containers:
+            - name: stage
+              emptyDir: {}
+          initContainers:
             - name: immich-go
               image: "src.thehellings.com/greg/immich:latest"
               imagePullPolicy: "Always"
@@ -46,18 +70,49 @@ spec:
                   immich-go
                   archive
                   "--write-to-folder"
-                  /nas1
+                  /stage
                   from-immich
                   "--from-api-key"
                   "$IMMICH_API_KEY"
                   "--from-server"
                   "http://immich-server:2283"
               volumeMounts:
-                - name: nas1
-                  mountPath: /nas1
+                - name: stage
+                  mountPath: /stage
               env:
                 - name: IMMICH_API_KEY
                   valueFrom:
                     secretKeyRef:
                       name: immich-login
                       key: apikey
+          containers:
+            - name: rclone
+              image: "rclone/rclone:latest"
+              command:
+                - /bin/sh
+                - "-c"
+                - |
+                  rclone sync --checksum \
+                    --s3-provider Minio \
+                    --s3-endpoint "$MINIO_ENDPOINT" \
+                    --s3-access-key-id "$MINIO_ACCESS_KEY" \
+                    --s3-secret-access-key "$MINIO_SECRET_KEY" \
+                    --s3-no-check-bucket \
+                    /stage \
+                    :s3:immich
+              volumeMounts:
+                - name: stage
+                  mountPath: /stage
+              env:
+                - name: MINIO_ENDPOINT
+                  value: "http://nas1.shire-zebra.ts.net:9000"
+                - name: MINIO_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: immich-minio-creds
+                      key: minio_key
+                - name: MINIO_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: immich-minio-creds
+                      key: minio_secret