From 58a5c95c32f13fcfdebd2c72bb1368afb9e7a0bc Mon Sep 17 00:00:00 2001 From: klaatu Date: Sat, 4 Apr 2026 06:12:31 +0000 Subject: [PATCH 1/5] feat(hosea): add Restic backup for AlbyHub /chain/alby --- hosts/unstable/hosea/default.nix | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/hosts/unstable/hosea/default.nix b/hosts/unstable/hosea/default.nix index 537be9c..d84aa92 100644 --- a/hosts/unstable/hosea/default.nix +++ b/hosts/unstable/hosea/default.nix @@ -145,6 +145,24 @@ in }; }; prometheus.exporters.graphite.enable = true; + restic.backups.albyhub = { + # AlbyHub LDK node data — must not be snapshotted live + paths = [ "/chain/alby" ]; + environmentFile = config.age.secrets.restic-env.path; + passwordFile = config.age.secrets.restic-pw.path; + initialize = true; + pruneOpts = [ + "--keep-daily 7" + "--keep-weekly 4" + "--keep-monthly 12" + ]; + backupPrepareCommand = "systemctl stop albyhub || true"; + backupCleanupCommand = "systemctl start albyhub"; + timerConfig = { + OnCalendar = "02:30"; + RandomizedDelaySec = "30min"; + }; + }; # Configure keymap xserver.xkb = { layout = "us"; @@ -158,6 +176,12 @@ in file = ../../../secrets/grafana-api-token.age; owner = "grafana"; }; + age.secrets.restic-pw = { + file = ../../../secrets/restic-pw.age; + }; + age.secrets.restic-env = { + file = ../../../secrets/restic-env.age; + }; environment.etc = { "grafana-dashboards/system-health.json".text = '' -- 2.54.0 From c74d647ec305347c631e5f235d59800e8dfca26c Mon Sep 17 00:00:00 2001 From: klaatu Date: Sat, 4 Apr 2026 06:12:56 +0000 Subject: [PATCH 2/5] feat(gitea): add daily dump CronJob writing to NAS1 NFS --- manifests/gitea/dump-cronjob.yaml | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 manifests/gitea/dump-cronjob.yaml diff --git a/manifests/gitea/dump-cronjob.yaml b/manifests/gitea/dump-cronjob.yaml new file mode 100644 index 0000000..0e43f63 --- /dev/null +++ b/manifests/gitea/dump-cronjob.yaml @@ -0,0 +1,61 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: gitea-dump +spec: + schedule: "0 3 * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + volumes: + - name: gitea-data + persistentVolumeClaim: + claimName: gitea-shared-storage + - name: dump-output + nfs: + path: /mnt/all/backups/gitea-dumps + server: nas1.shire-zebra.ts.net + initContainers: + - name: gitea-dump + image: "gitea/gitea:1.25.4" + command: + - /bin/sh + - "-c" + - | + set -e + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + OUTFILE="/dump-output/gitea-dump-${TIMESTAMP}.zip" + gitea dump \ + --config /data/gitea/conf/app.ini \ + --file "${OUTFILE}" \ + --type zip \ + --skip-log \ + --skip-custom-dir \ + --skip-db + echo "Dump written to ${OUTFILE}" + volumeMounts: + - name: gitea-data + mountPath: /data + readOnly: true + - name: dump-output + mountPath: /dump-output + containers: + - name: cleanup + image: "busybox:1.36" + command: + - /bin/sh + - "-c" + - | + ls -t /dump-output/gitea-dump-*.zip 2>/dev/null | tail -n +31 | xargs rm -f + echo "Cleanup done." + volumeMounts: + - name: dump-output + mountPath: /dump-output -- 2.54.0 From 6b84824a53efade91203e323df099555a0770e6f Mon Sep 17 00:00:00 2001 From: klaatu Date: Sat, 4 Apr 2026 06:13:00 +0000 Subject: [PATCH 3/5] feat(gitea): add dump-cronjob to kustomization --- manifests/gitea/kustomization.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/gitea/kustomization.yaml b/manifests/gitea/kustomization.yaml index e48af02..a66546d 100644 --- a/manifests/gitea/kustomization.yaml +++ b/manifests/gitea/kustomization.yaml @@ -6,3 +6,4 @@ resources: - chart.yaml - ingress.yaml - secrets.yaml + - dump-cronjob.yaml -- 2.54.0 From 1077d357cacb8979296b87e6c1655c35b83b3dd1 Mon Sep 17 00:00:00 2001 From: klaatu Date: Sat, 4 Apr 2026 06:13:43 +0000 Subject: [PATCH 4/5] =?UTF-8?q?feat(immich):=20switch=20backup=20to=20rclo?= =?UTF-8?q?ne=20sync=20=E2=86=92=20MinIO=20immich=20bucket=20on=20nas1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- manifests/immich/backup.yaml | 89 ++++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/manifests/immich/backup.yaml b/manifests/immich/backup.yaml index c5cfe5b..35e9b43 100644 --- a/manifests/immich/backup.yaml +++ b/manifests/immich/backup.yaml @@ -19,23 +19,76 @@ spec: key: "dd0de40c-9ad6-4034-9d5c-b311016ebf19" property: apikey --- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: immich-minio-creds +spec: + target: + name: immich-minio-creds + deletionPolicy: Delete + template: + type: Opaque + data: + access-key: "{{ .minio_key }}" + secret-key: "{{ .minio_secret }}" + rclone.conf: | + [nas1minio] + type = s3 + provider = Minio + endpoint = http://nas1.shire-zebra.ts.net:9000 + access_key_id = {{ .minio_key }} + secret_access_key = {{ .minio_secret }} + region = us-east-1 + secretStoreRef: + name: bitwarden-login + kind: ClusterSecretStore + data: + - secretKey: minio_key + remoteRef: + key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d + property: username + - secretKey: minio_secret + remoteRef: + key: dcbcf704-7dce-48d7-bbd1-b3a801875b3d + property: password +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: immich-backup-staging +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-default + resources: + requests: + storage: 400Gi +--- apiVersion: batch/v1 kind: CronJob metadata: name: backup spec: schedule: "0 4 * * *" # At 4am each day + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 jobTemplate: spec: template: spec: restartPolicy: Never volumes: - - name: nas1 - nfs: - path: /mnt/all/backups/immich - server: nas1.shire-zebra.ts.net - containers: + - name: staging + persistentVolumeClaim: + claimName: immich-backup-staging + - name: rclone-config + secret: + secretName: immich-minio-creds + items: + - key: rclone.conf + path: rclone.conf + initContainers: - name: immich-go image: "src.thehellings.com/greg/immich:latest" imagePullPolicy: "Always" @@ -46,18 +99,38 @@ spec: immich-go archive "--write-to-folder" - /nas1 + /staging from-immich "--from-api-key" "$IMMICH_API_KEY" "--from-server" "http://immich-server:2283" volumeMounts: - - name: nas1 - mountPath: /nas1 + - name: staging + mountPath: /staging env: - name: IMMICH_API_KEY valueFrom: secretKeyRef: name: immich-login key: apikey + containers: + - name: rclone-sync + image: "rclone/rclone:1.68" + command: + - /bin/sh + - "-c" + - | + rclone sync \ + --config /rclone-config/rclone.conf \ + --checksum \ + --progress \ + --transfers 4 \ + --checkers 8 \ + /staging nas1minio:immich + volumeMounts: + - name: staging + mountPath: /staging + - name: rclone-config + mountPath: /rclone-config + readOnly: true -- 2.54.0 From 02184470080533414b3df9d7bbb80a8930b8e24e Mon Sep 17 00:00:00 2001 From: Klaatu Date: Sat, 4 Apr 2026 02:05:03 -0500 Subject: [PATCH 5/5] fix: address review feedback on backup improvements - manifests/gitea/dump-cronjob.yaml: Remove --skip-log, --skip-custom-dir, --skip-db flags to make backup complete - manifests/gitea/dump-cronjob.yaml: Replace NFS volume + cleanup container with S3 upload to backup-gitea bucket using MinIO client (mc). 30-day lifecycle set via mc ilm. Uses minio_key/minio_secret from existing gitea-config secret. - hosts/unstable/hosea/default.nix: Replace raw services.restic.backups.albyhub block with greg.backup.jobs.albyhub using the greg.backup module. Remove manual age.secrets.restic-env and age.secrets.restic-pw entries since the greg.backup module declares them. --- hosts/unstable/hosea/default.nix | 30 ++++--------------- manifests/gitea/dump-cronjob.yaml | 49 ++++++++++++++++++++----------- 2 files changed, 38 insertions(+), 41 deletions(-) diff --git a/hosts/unstable/hosea/default.nix b/hosts/unstable/hosea/default.nix index d84aa92..c98c4a4 100644 --- a/hosts/unstable/hosea/default.nix +++ b/hosts/unstable/hosea/default.nix @@ -60,6 +60,12 @@ in enable = true; tags = [ "home" ]; }; + backup.jobs.albyhub = { + src = "/chain/alby"; + dest = "albyhub"; + pre = "systemctl stop albyhub || true"; + post = "systemctl start albyhub"; + }; }; hardware = { @@ -145,24 +151,6 @@ in }; }; prometheus.exporters.graphite.enable = true; - restic.backups.albyhub = { - # AlbyHub LDK node data — must not be snapshotted live - paths = [ "/chain/alby" ]; - environmentFile = config.age.secrets.restic-env.path; - passwordFile = config.age.secrets.restic-pw.path; - initialize = true; - pruneOpts = [ - "--keep-daily 7" - "--keep-weekly 4" - "--keep-monthly 12" - ]; - backupPrepareCommand = "systemctl stop albyhub || true"; - backupCleanupCommand = "systemctl start albyhub"; - timerConfig = { - OnCalendar = "02:30"; - RandomizedDelaySec = "30min"; - }; - }; # Configure keymap xserver.xkb = { layout = "us"; @@ -176,12 +164,6 @@ in file = ../../../secrets/grafana-api-token.age; owner = "grafana"; }; - age.secrets.restic-pw = { - file = ../../../secrets/restic-pw.age; - }; - age.secrets.restic-env = { - file = ../../../secrets/restic-env.age; - }; environment.etc = { "grafana-dashboards/system-health.json".text = '' diff --git a/manifests/gitea/dump-cronjob.yaml b/manifests/gitea/dump-cronjob.yaml index 0e43f63..9e40b67 100644 --- a/manifests/gitea/dump-cronjob.yaml +++ b/manifests/gitea/dump-cronjob.yaml @@ -19,10 +19,8 @@ spec: - name: gitea-data persistentVolumeClaim: claimName: gitea-shared-storage - - name: dump-output - nfs: - path: /mnt/all/backups/gitea-dumps - server: nas1.shire-zebra.ts.net + - name: dump-staging + emptyDir: {} initContainers: - name: gitea-dump image: "gitea/gitea:1.25.4" @@ -32,30 +30,47 @@ spec: - | set -e TIMESTAMP=$(date +%Y%m%d-%H%M%S) - OUTFILE="/dump-output/gitea-dump-${TIMESTAMP}.zip" + OUTFILE="/dump-staging/gitea-dump-${TIMESTAMP}.zip" gitea dump \ --config /data/gitea/conf/app.ini \ --file "${OUTFILE}" \ - --type zip \ - --skip-log \ - --skip-custom-dir \ - --skip-db + --type zip echo "Dump written to ${OUTFILE}" volumeMounts: - name: gitea-data mountPath: /data readOnly: true - - name: dump-output - mountPath: /dump-output + - name: dump-staging + mountPath: /dump-staging containers: - - name: cleanup - image: "busybox:1.36" + - name: upload-to-s3 + image: "minio/mc:latest" command: - /bin/sh - "-c" - | - ls -t /dump-output/gitea-dump-*.zip 2>/dev/null | tail -n +31 | xargs rm -f - echo "Cleanup done." + set -e + # Configure mc alias for MinIO + mc alias set nas1 http://nas1.shire-zebra.ts.net:9000 \ + "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" + # Upload dump to backup-gitea bucket + DUMP_FILE=$(ls /dump-staging/gitea-dump-*.zip | head -1) + mc cp "${DUMP_FILE}" "nas1/backup-gitea/$(basename ${DUMP_FILE})" + echo "Uploaded $(basename ${DUMP_FILE}) to backup-gitea" + # Set 30-day lifecycle on the bucket (idempotent) + mc ilm rule add --expire-days 30 nas1/backup-gitea 2>/dev/null || true volumeMounts: - - name: dump-output - mountPath: /dump-output + - name: dump-staging + mountPath: /dump-staging + readOnly: true + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: gitea-config + key: minio_key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: gitea-config + key: minio_secret -- 2.54.0