Upgrade Immich to v2.1.0

Move to using kustomize'd helm charts to deploy it
Update the new version of the helm chart, which has very different
values structure
Move database into the db namespace
Upgrade the db to include vectorchord
Upgrade CNPG to now include barman cloud natively
Migrate databases to Barman Cloud plugin, completely
Eliminate yet another shell script step in applying my k8s config

manifests/apply.sh

@@ -18,5 +18,4 @@ helm upgrade --install cnpg \
 sleep 5
 kubectl apply -k .
 
-./immich/apply.sh
 ./tailscale/apply.sh

manifests/cnpg-system/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: cnpg-system
+
+resources:
+  # https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/v0.7.0/manifest.yaml
+  - barman-cloud.yaml

manifests/databases/immich.yaml

@@ -0,0 +1,89 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: immich
+spec:
+  imageName: "ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration"
+  #imageName: "ghcr.io/tensorchord/cloudnative-pgvecto.rs:16-v0.3.0"
+  #postgresUID: 1
+  instances: 1
+  storage:
+    size: 60Gi
+  primaryUpdateStrategy: unsupervised
+  postgresql:
+    shared_preload_libraries:
+      - vectors.so
+      - vchord.so
+
+  bootstrap:
+    recovery:
+      source: origin
+    # initdb:
+    #   database: immich
+    #   owner: immich
+    #   secret:
+    #     name: postgres-user-immich
+    #   dataChecksums: true
+    #   postInitApplicationSQL:
+    #     - ALTER SYSTEM SET search_path TO "$user", public, vectors;
+    #     - SET search_path TO "$user", public, vectors;
+    #     - CREATE EXTENSION IF NOT EXISTS "cube";
+    #     - CREATE EXTENSION IF NOT EXISTS "earthdistance";
+    #     - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "immich";
+
+  externalClusters:
+    - name: origin
+      plugin:
+        name: barman-cloud.cloudnative-pg.io
+        parameters:
+          barmanObjectName: k3sbackup-objectstore
+          serverName: pgvector
+
+  managed:
+    roles:
+      - name: immich
+        ensure: present
+        comment: Immich DB user
+        login: true
+        superuser: true
+        passwordSecret:
+          name: postgres-user-immich
+
+  # plugins:
+  #   - name: barman-cloud.cloudnative-pg.io
+  #     isWALArchiver: true
+  #     parameters:
+  #       barmanObjectName: k3sbackup-objectstore
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  name: database-immich
+spec:
+  name: immich
+  owner: immich
+  cluster:
+    name: immich
+  extensions:
+    - name: vectors
+      ensure: present
+    - name: vectorchord
+      ensure: present
+    - name: cube
+      ensure: present
+    - name: earthdistance
+      ensure: present
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: immich-backup
+spec:
+  immediate: true # Create one when this is added to the cluster
+  schedule: "0 2 0 * * *" # 1AM, nightly
+  backupOwnerReference: self
+  cluster:
+    name: immich
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

manifests/databases/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - postgres-gitlab.yaml
   - postgres-pgadmin.yaml
   - postgres-matrix.yaml
+  - immich.yaml
   - ingress.yaml

manifests/databases/postgres-cluster.yaml

@@ -35,7 +35,7 @@ spec:
 
   plugins:
     - name: barman-cloud.cloudnative-pg.io
-      isWALArchiver: false
+      isWALArchiver: true
       parameters:
         barmanObjectName: k3sbackup-objectstore
 
@@ -85,3 +85,6 @@ spec:
   backupOwnerReference: self
   cluster:
     name: postgres
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

manifests/databases/secrets.yaml

@@ -133,3 +133,37 @@ spec:
       remoteRef:
         key: 685b29c6-9264-4e60-ba4a-b2ea005a5d7b
         property: password
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: postgres-user-immich
+  namespace: db
+spec:
+  target:
+    name: postgres-user-immich
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
+        property: password

manifests/immich/apply.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-
-# Get the directory where the script is located
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-# Change to the script directory
-cd "$SCRIPT_DIR"
-
-kubectl apply -k "$SCRIPT_DIR"
-
-# https://www.dragonflydb.io/guides/redis-kubernetes
-# Deploys into immich namespace, directly, in order to allow the password to be
-# accessed by the immich installer
-helm upgrade --install --create-namespace --namespace immich redis \
-    oci://registry-1.docker.io/bitnamicharts/redis \
-    -f "${SCRIPT_DIR}/values-redis.yaml" \
-    --wait
-# https://github.com/immich-app/immich-charts/tree/main
-helm upgrade --install --create-namespace --namespace immich immich \
-    oci://ghcr.io/immich-app/immich-charts/immich \
-    -f "${SCRIPT_DIR}/values.yaml" \
-    --wait

manifests/immich/chart.yaml

@@ -0,0 +1,77 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: OCIRepository
+metadata:
+  name: immich
+spec:
+  interval: "24h"
+  url: oci://ghcr.io/immich-app/immich-charts/immich
+  ref:
+    # Version 0.10.0
+    # For some reason, I can't get it to pull by tag or by auto-discovery
+    digest: "sha256:bcffd3d504664710baa7c0bcd66dc246437c51a9a76f9b8ed1a10c7aecd80766"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: immich
+spec:
+  interval: "1h"
+  chartRef:
+    kind: OCIRepository
+    name: immich
+    namespace: immich
+  dependsOn:
+    - name: longhorn
+      namespace: longhorn-system
+  values:
+    controllers:
+      main:
+        containers:
+          main:
+            image:
+              tag: v2.1.0
+            env:
+              DB_HOSTNAME: immich-rw.db.svc.cluster.local
+              DB_DATABASE_NAME: immich
+              DB_USERNAME:
+                valueFrom:
+                  secretKeyRef:
+                    name: postgres-user-immich
+                    key: username
+              DB_PASSWORD:
+                valueFrom:
+                  secretKeyRef:
+                    name: postgres-user-immich
+                    key: password
+    immich:
+      persistence:
+        library:
+          existingClaim: immich-storage
+    server:
+      ingress:
+        main:
+          enabled: true
+          annogations:
+            ingressClassName: traefik
+            # Set body size to 10G to allow uploading large things
+            traefik.ingress.kubernetes.io/buffering: |
+              maxrequestbodybytes: 10000000000
+              memrequestbodybytes: 20000000000
+          hosts:
+            - host: immich.cluster
+              paths:
+                - path: "/"
+    machine-learning:
+      persistence:
+        cache:
+          type: persistentVolumeClaim
+          storageClass: longhorn-default
+          size: 25Gi
+    valkey:
+      enabled: true
+      persistence:
+        data:
+          enabled: true
+          size: 2Gi
+          type: persistentVolumeClaim
+          storageClass: longhorn-default

manifests/immich/database.yaml

@@ -1,74 +0,0 @@
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  namespace: db
-  name: pgvector
-spec:
-  imageName: "ghcr.io/tensorchord/cloudnative-pgvecto.rs:16-v0.3.0"
-  instances: 1
-  storage:
-    size: 40Gi
-  primaryUpdateStrategy: unsupervised
-  postgresql:
-    shared_preload_libraries:
-      - "vectors.so"
-
-  bootstrap:
-    initdb:
-      database: immich
-      owner: immich
-      secret:
-        name: postgres-user-immich
-      dataChecksums: true
-      postInitApplicationSQL:
-        - ALTER SYSTEM SET search_path TO "$user", public, vectors;
-        - SET search_path TO "$user", public, vectors;
-        - CREATE EXTENSION IF NOT EXISTS "vectors";
-        - CREATE EXTENSION IF NOT EXISTS "cube";
-        - CREATE EXTENSION IF NOT EXISTS "earthdistance";
-        - ALTER SCHEMA vectors OWNER TO "immich";
-        - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA vectors TO "immich";
-        - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "immich";
-  managed:
-    roles:
-      - name: immich
-        ensure: present
-        comment: Immich DB user
-        login: true
-        superuser: false
-        passwordSecret:
-          name: postgres-user-immich
-  backup:
-    retentionPolicy: "30d"
-    barmanObjectStore:
-      destinationPath: "s3://k3sbackup/pgvector"
-      endpointURL: "http://s3.thehellings.lan:9000/"
-      s3Credentials:
-        accessKeyId:
-          name: k3sbackup
-          key: username
-        secretAccessKey:
-          name: k3sbackup
-          key: password
-      wal:
-        compression: gzip
----
-apiVersion: postgresql.cnpg.io/v1
-kind: Database
-metadata:
-  namespace: db
-  name: database-immich
-spec:
-  name: immich
-  owner: immich
-  cluster:
-    name: pgvector
-  extensions:
-    - name: vectors
-      ensure: present
-    - name: vectorchord
-      ensure: present
-    - name: cube
-      ensure: present
-    - name: earthdistance
-      ensure: present

manifests/immich/db-restore.yaml

@@ -0,0 +1,89 @@
+# This one needs to have a different name than the cluster it is being
+# restored from, so we create a restore cluster here and do not configure
+# it with any backup settings
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  namespace: db
+  name: pgvector-restore
+spec:
+  imageName: "ghcr.io/tensorchord/cloudnative-pgvecto.rs:16-v0.3.0"
+  instances: 1
+  storage:
+    size: 60Gi
+  primaryUpdateStrategy: unsupervised
+  postgresql:
+    shared_preload_libraries:
+      - "vectors.so"
+
+  bootstrap:
+    recovery:
+      source: origin
+  externalClusters:
+    - name: origin
+      plugin:
+        name: barman-cloud.cloudnative-pg.io
+        parameters:
+          barmanObjectName: k3sbackup-objectstore
+          serverName: pgvector
+  managed:
+    roles:
+      - name: immich
+        ensure: present
+        comment: Immich DB user
+        login: true
+        superuser: true
+        passwordSecret:
+          name: postgres-user-immich
+---
+# This cluster will stream from the above one, since we cannot rename
+# objects in Kubernetes
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  namespace: db
+  name: pgvector
+spec:
+  imageName: "ghcr.io/immich-app/postgres:16-vectorchord0.5.3"
+  #imageName: "ghcr.io/tensorchord/cloudnative-pgvecto.rs:16-v0.3.0"
+  instances: 1
+  storage:
+    size: 60Gi
+  primaryUpdateStrategy: unsupervised
+  postgresUID: 26
+  postgresql:
+    shared_preload_libraries:
+      - "vectors.so"
+
+  bootstrap:
+    pg_basebackup:
+      source: pgvector-restore
+  externalClusters:
+    - name: pgvector-restore
+      connectionParameters:
+        host: pgvector-restore-rw
+        user: streaming_replica
+        sslmode: verify-full
+      sslKey:
+        name: pgvector-restore-replication
+        key: tls.key
+      sslCert:
+        name: pgvector-restore-replication
+        key: tls.crt
+      sslRootCert:
+        name: pgvector-restore-ca
+        key: ca.crt
+  managed:
+    roles:
+      - name: immich
+        ensure: present
+        comment: Immich DB user
+        login: true
+        superuser: true
+        passwordSecret:
+          name: postgres-user-immich
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: k3sbackup-objectstore

manifests/immich/kustomization.yaml

@@ -1,6 +1,8 @@
+namespace: immich
+
 resources:
   - namespace.yaml
+  - chart.yaml
   - postgres-user-secret.yaml
-  - database.yaml
   - pvc.yaml
   - ingress.yaml

manifests/immich/postgres-user-secret.yaml

@@ -2,41 +2,6 @@ apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
   name: postgres-user-immich
-  namespace: db
-spec:
-  target:
-    name: postgres-user-immich
-    deletionPolicy: Delete
-    template:
-      type: Opaque
-      data:
-        username: |-
-          {{ .username }}
-        password: |-
-          {{ .password }}
-  data:
-    - secretKey: username
-      sourceRef:
-        storeRef:
-          name: bitwarden-login
-          kind: ClusterSecretStore
-      remoteRef:
-        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
-        property: username
-    - secretKey: password
-      sourceRef:
-        storeRef:
-          name: bitwarden-login
-          kind: ClusterSecretStore
-      remoteRef:
-        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
-        property: password
----
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: postgres-user-immich
-  namespace: immich
 spec:
   target:
     name: postgres-user-immich

manifests/immich/values-redis.yaml

@@ -1,8 +0,0 @@
-# https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml
-# https://github.com/bitnami/charts/tree/main/bitnami/redis
-architecture: standalone
-global:
-  defaultStorageClass: longhorn-default
-master:
-  persistence:
-    storageClass: longhorn-default

manifests/immich/values.yaml

@@ -1,46 +0,0 @@
-# https://github.com/immich-app/immich-charts/blob/main/charts/immich/values.yaml
-env:
-  DB_HOSTNAME: pgvector-rw.db.svc.cluster.local
-  DB_DATABASE_NAME: immich
-  DB_USERNAME:
-    valueFrom:
-      secretKeyRef:
-        name: postgres-user-immich
-        key: username
-  DB_PASSWORD:
-    valueFrom:
-      secretKeyRef:
-        name: postgres-user-immich
-        key: password
-  REDIS_HOSTNAME: redis-master
-  REDIS_PASSWORD:
-    valueFrom:
-      secretKeyRef:
-        name: redis
-        key: redis-password
-image:
-  tag: "v1.135.3"
-immich:
-  persistence:
-    library:
-      existingClaim: immich-storage
-server:
-  ingress:
-    main:
-      enabled: true
-      annogations:
-        ingressClassName: traefik
-        # Set body size to 10G to allow uploading large things
-        traefik.ingress.kubernetes.io/buffering: |
-          maxrequestbodybytes: 10000000000
-          memrequestbodybytes: 20000000000
-      hosts:
-        - host: immich.cluster
-          paths:
-            - path: "/"
-machine-learning:
-  persistence:
-    cache:
-      type: pvc
-      storageClass: longhorn-default
-      size: 25Gi

manifests/kustomization.yaml

@@ -2,6 +2,7 @@ resources:
   - helm
   - bitwarden
   - secrets
+  - cnpg-system
   - databases
   - matrix
   - gitlab-runner