hosts/unstable/hosea/default.nix

@@ -60,6 +60,12 @@ in
       enable = true;
       tags = [ "home" ];
     };
+    backup.jobs.albyhub = {
+      src = "/chain/alby";
+      dest = "albyhub";
+      pre = "systemctl stop albyhub || true";
+      post = "systemctl start albyhub";
+    };
   };
 
   hardware = {

manifests/gitea/dump-cronjob.yaml

@@ -0,0 +1,76 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitea-dump
+spec:
+  schedule: "0 3 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsUser: 1000
+            runAsGroup: 1000
+            fsGroup: 1000
+          volumes:
+            - name: gitea-data
+              persistentVolumeClaim:
+                claimName: gitea-shared-storage
+            - name: dump-staging
+              emptyDir: {}
+          initContainers:
+            - name: gitea-dump
+              image: "gitea/gitea:1.25.4"
+              command:
+                - /bin/sh
+                - "-c"
+                - |
+                  set -e
+                  TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+                  OUTFILE="/dump-staging/gitea-dump-${TIMESTAMP}.zip"
+                  gitea dump \
+                    --config /data/gitea/conf/app.ini \
+                    --file "${OUTFILE}" \
+                    --type zip
+                  echo "Dump written to ${OUTFILE}"
+              volumeMounts:
+                - name: gitea-data
+                  mountPath: /data
+                  readOnly: true
+                - name: dump-staging
+                  mountPath: /dump-staging
+          containers:
+            - name: upload-to-s3
+              image: "minio/mc:latest"
+              command:
+                - /bin/sh
+                - "-c"
+                - |
+                  set -e
+                  # Configure mc alias for MinIO
+                  mc alias set nas1 http://nas1.shire-zebra.ts.net:9000 \
+                    "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
+                  # Upload dump to backup-gitea bucket
+                  DUMP_FILE=$(ls /dump-staging/gitea-dump-*.zip | head -1)
+                  mc cp "${DUMP_FILE}" "nas1/backup-gitea/$(basename ${DUMP_FILE})"
+                  echo "Uploaded $(basename ${DUMP_FILE}) to backup-gitea"
+                  # Set 30-day lifecycle on the bucket (idempotent)
+                  mc ilm rule add --expire-days 30 nas1/backup-gitea 2>/dev/null || true
+              volumeMounts:
+                - name: dump-staging
+                  mountPath: /dump-staging
+                  readOnly: true
+              env:
+                - name: MINIO_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: gitea-config
+                      key: minio_key
+                - name: MINIO_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: gitea-config
+                      key: minio_secret

manifests/gitea/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - chart.yaml
   - ingress.yaml
   - secrets.yaml
+  - dump-cronjob.yaml

manifests/immich/backup.yaml

@@ -19,23 +19,76 @@ spec:
         key: "dd0de40c-9ad6-4034-9d5c-b311016ebf19"
         property: apikey
 ---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: immich-minio-creds
+spec:
+  target:
+    name: immich-minio-creds
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        access-key: "{{ .minio_key }}"
+        secret-key: "{{ .minio_secret }}"
+        rclone.conf: |
+          [nas1minio]
+          type = s3
+          provider = Minio
+          endpoint = http://nas1.shire-zebra.ts.net:9000
+          access_key_id = {{ .minio_key }}
+          secret_access_key = {{ .minio_secret }}
+          region = us-east-1
+  secretStoreRef:
+    name: bitwarden-login
+    kind: ClusterSecretStore
+  data:
+    - secretKey: minio_key
+      remoteRef:
+        key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d
+        property: username
+    - secretKey: minio_secret
+      remoteRef:
+        key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d
+        property: password
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-backup-staging
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-default
+  resources:
+    requests:
+      storage: 400Gi
+---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: backup
 spec:
   schedule: "0 4 * * *" # At 4am each day
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
   jobTemplate:
     spec:
       template:
         spec:
           restartPolicy: Never
           volumes:
-            - name: nas1
+            - name: staging
-              nfs:
+              persistentVolumeClaim:
-                path: /mnt/all/backups/immich
+                claimName: immich-backup-staging
-                server: nas1.shire-zebra.ts.net
+            - name: rclone-config
-          containers:
+              secret:
+                secretName: immich-minio-creds
+                items:
+                  - key: rclone.conf
+                    path: rclone.conf
+          initContainers:
             - name: immich-go
               image: "src.thehellings.com/greg/immich:latest"
               imagePullPolicy: "Always"
@@ -46,18 +99,38 @@ spec:
                   immich-go
                   archive
                   "--write-to-folder"
-                  /nas1
+                  /staging
                   from-immich
                   "--from-api-key"
                   "$IMMICH_API_KEY"
                   "--from-server"
                   "http://immich-server:2283"
               volumeMounts:
-                - name: nas1
+                - name: staging
-                  mountPath: /nas1
+                  mountPath: /staging
               env:
                 - name: IMMICH_API_KEY
                   valueFrom:
                     secretKeyRef:
                       name: immich-login
                       key: apikey
+          containers:
+            - name: rclone-sync
+              image: "rclone/rclone:1.68"
+              command:
+                - /bin/sh
+                - "-c"
+                - |
+                  rclone sync \
+                    --config /rclone-config/rclone.conf \
+                    --checksum \
+                    --progress \
+                    --transfers 4 \
+                    --checkers 8 \
+                    /staging nas1minio:immich
+              volumeMounts:
+                - name: staging
+                  mountPath: /staging
+                - name: rclone-config
+                  mountPath: /rclone-config
+                  readOnly: true