Enable Immich

Kubernetes files to stand up Immich, including Redis, a separate
Postgres cluster, and more

hosts/genesis/net/hosts

@@ -24,7 +24,7 @@
 10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan
 
 # VIP
-10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster
+10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster immich.cluster
 
 # IPMI
 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan
@@ -37,7 +37,7 @@
 100.91.131.66 gitlab.home gitlab.shire-zebra.ts.net gitlab.thehellings.lan registry.thehellings.lan git.thehellings.lan
 100.68.203.1 hosea.home hosea.shire-zebra.ts.net
 100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes longhorn.kubernetes
-100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes
+100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes immich.kubernetes
 100.90.74.19 zeke.home
 100.115.57.8 linode.home
 100.65.5.38 matrix.home matrix.shire-zebra.ts.net

manifests/apply.sh

@@ -11,14 +11,14 @@ for n in isaiah jeremiah zeke; do
     kubectl label nodes "${n}" "node.longhorn.io/create-default-disk=config"
 done
 # Now, configure longhorn settings for each node
-kubectl annotate nodes isaiah 'node.longhorn.io/default-disks-config=[
+kubectl annotate nodes --overwrite isaiah 'node.longhorn.io/default-disks-config=[
-  { "path": "/var/lib/longhorn", "allowScheduling" : true }
+  { "path": "/var/lib/longhorn", "allowScheduling" : true, "tags": ["hdd", "large"]}
 ]'
-kubectl annotate nodes jeremiah 'node.longhorn.io/default-disks-config=[
+kubectl annotate nodes --overwrite jeremiah 'node.longhorn.io/default-disks-config=[
-  { "path": "/var/lib/longhorn", "allowScheduling" : true }
+  { "path": "/var/lib/longhorn", "allowScheduling" : true, "tags": ["hdd", "large"]}
 ]'
-kubectl annotate nodes zeke 'node.longhorn.io/default-disks-config=[
+kubectl annotate nodes --overwrite zeke 'node.longhorn.io/default-disks-config=[
-  { "path": "/var/lib/longhorn", "allowScheduling" : true }
+  { "path": "/var/lib/longhorn", "allowScheduling" : trues, "tags": ["ssd", "fast"]}
 ]'
 
 kubectl apply -k namespaces
@@ -29,3 +29,13 @@ sleep 15
 kubectl apply -k helm
 sleep 5
 kubectl apply -k .
+sleep 5
+# https://cloudnative-pg.io
+helm repo add cnpg https://cloudnative-pg.github.io/charts/
+helm upgrade --install cnpg \
+    --create-namespace --namespace cnpg-system \
+    cnpg/cloudnative-pg \
+    -f values/cnpg.yaml \
+    --wait
+
+./immich/apply.sh

manifests/helm/cloudnative-pg.yaml

@@ -1,33 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: cnpg-system
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: cloudnative-pg
-  namespace: cnpg-system
-spec:
-  interval: "24h"
-  url: "https://cloudnative-pg.github.io/charts/"
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: cnpg
-  namespace: cnpg-system
-spec:
-  interval: 10m
-  chart:
-    spec:
-      chart: cloudnative-pg
-      version: "0.23.2"
-      sourceRef:
-        kind: HelmRepository
-        name: cloudnative-pg
-      interval: "1h"
-  values:
-    crds:
-      create: true
-    includeCRDs: true

manifests/helm/kustomization.yaml

@@ -4,4 +4,3 @@ resources:
   - longhorn.yaml  # Needed for storage
   - traefik.yaml
   - external-secrets.yaml
-  - cloudnative-pg.yaml

manifests/helm/longhorn.yaml

@@ -113,3 +113,35 @@ spec:
               number: 80
   - <<: *host
     host: longhorn.kubernetes
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-default
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+parameters:
+  backupTargetName: default
+  numberOfReplicas: "2"
+  staleReplicaTimeout: "2880"
+  fromBackup: ""
+  fsType: ext4
+  diskSelector: "hdd,large"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-fast
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+parameters:
+  backupTargetName: default
+  numberOfReplicas: "1"
+  staleReplicaTimeout: "2880"
+  fromBackup: ""
+  fsType: ext4
+  diskSelector: "ssd,fast"

manifests/immich/apply.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+
+# Get the directory where the script is located
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+# Change to the script directory
+cd "$SCRIPT_DIR"
+
+kubectl apply -k "$SCRIPT_DIR"
+
+# https://www.dragonflydb.io/guides/redis-kubernetes
+# Deploys into immich namespace, directly, in order to allow the password to be
+# accessed by the immich installer
+helm upgrade --install --create-namespace --namespace immich redis \
+    oci://registry-1.docker.io/bitnamicharts/redis \
+    -f "${SCRIPT_DIR}/values-redis.yaml" \
+    --wait
+# https://github.com/immich-app/immich-charts/tree/main
+helm upgrade --install --create-namespace --namespace immich immich \
+    oci://ghcr.io/immich-app/immich-charts/immich \
+    -f "${SCRIPT_DIR}/values.yaml" \
+    --wait

manifests/immich/database.yaml

@@ -0,0 +1,74 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  namespace: db
+  name: pgvector
+spec:
+  imageName: "ghcr.io/tensorchord/cloudnative-pgvecto.rs:16-v0.3.0"
+  instances: 1
+  storage:
+    size: 10Gi
+  primaryUpdateStrategy: unsupervised
+  postgresql:
+    shared_preload_libraries:
+      - "vectors.so"
+
+  bootstrap:
+    initdb:
+      database: immich
+      owner: immich
+      secret:
+        name: postgres-user-immich
+      dataChecksums: true
+      postInitApplicationSQL:
+        - ALTER SYSTEM SET search_path TO "$user", public, vectors;
+        - SET search_path TO "$user", public, vectors;
+        - CREATE EXTENSION IF NOT EXISTS "vectors";
+        - CREATE EXTENSION IF NOT EXISTS "cube";
+        - CREATE EXTENSION IF NOT EXISTS "earthdistance";
+        - ALTER SCHEMA vectors OWNER TO "immich";
+        - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA vectors TO "immich";
+        - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "immich";
+  managed:
+    roles:
+      - name: immich
+        ensure: present
+        comment: Immich DB user
+        login: true
+        superuser: false
+        passwordSecret:
+          name: postgres-user-immich
+  backup:
+    retentionPolicy: "30d"
+    barmanObjectStore:
+      destinationPath: "s3://k3sbackup/pgvector"
+      endpointURL: "http://s3.thehellings.lan:9000/"
+      s3Credentials:
+        accessKeyId:
+          name: k3sbackup
+          key: username
+        secretAccessKey:
+          name: k3sbackup
+          key: password
+      wal:
+        compression: gzip
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Database
+metadata:
+  namespace: db
+  name: database-immich
+spec:
+  name: immich
+  owner: immich
+  cluster:
+    name: pgvector
+  extensions:
+    - name: vectors
+      ensure: present
+    - name: vectorchord
+      ensure: present
+    - name: cube
+      ensure: present
+    - name: earthdistance
+      ensure: present

manifests/immich/ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: immich
+  name: immich
+  annotations:
+    ingressClassName: traefik
+    # Set body size to 10G to allow uploading large things
+    traefik.ingress.kubernetes.io/buffering: |
+      maxrequestbodybytes: 10000000000
+      memrequestbodybytes: 20000000000
+spec:
+  rules:
+    - &host
+      host: immich.kubernetes
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: immich-server
+                port:
+                  name: http
+    - <<: *host
+      host: immich.thehellings.com

manifests/immich/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - namespace.yaml
+  - postgres-user-secret.yaml
+  - database.yaml
+  - pvc.yaml
+  - ingress.yaml

manifests/immich/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: immich

manifests/immich/postgres-user-secret.yaml

@@ -0,0 +1,67 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: postgres-user-immich
+  namespace: db
+spec:
+  target:
+    name: postgres-user-immich
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
+        property: password
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: postgres-user-immich
+  namespace: immich
+spec:
+  target:
+    name: postgres-user-immich
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f
+        property: password

manifests/immich/pvc.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: immich
+  name: immich-storage
+spec:
+  storageClassName: longhorn-default
+  volumeName: immich-storage
+  resources:
+    requests:
+      storage: 250Gi
+  accessModes:
+    - ReadWriteOnce
+---
+#apiVersion: v1
+#kind: PersistentVolume
+#metadata:
+  #namespace: immich
+  #name: immich-storage
+#spec:
+  #storageClassName: longhorn-default
+  #capacity:
+    #storage: 250Gi
+  #volumeMode: Filesystem
+  #accessModes:
+    #- ReadWriteOnce
+  #csi:
+    #driver: driver.longhorn.io
+    #volumeHandle: immich-storage

manifests/immich/values-redis.yaml

@@ -0,0 +1,8 @@
+# https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml
+# https://github.com/bitnami/charts/tree/main/bitnami/redis
+architecture: standalone
+global:
+  defaultStorageClass: longhorn-default
+master:
+  persistence:
+    storageClass: longhorn-default

manifests/immich/values.yaml

@@ -0,0 +1,46 @@
+# https://github.com/immich-app/immich-charts/blob/main/charts/immich/values.yaml
+env:
+  DB_HOSTNAME: pgvector-rw.db.svc.cluster.local
+  DB_DATABASE_NAME: immich
+  DB_USERNAME:
+    valueFrom:
+      secretKeyRef:
+        name: postgres-user-immich
+        key: username
+  DB_PASSWORD:
+    valueFrom:
+      secretKeyRef:
+        name: postgres-user-immich
+        key: password
+  REDIS_HOSTNAME: redis-master
+  REDIS_PASSWORD:
+    valueFrom:
+      secretKeyRef:
+        name: redis
+        key: redis-password
+image:
+  tag: "v1.135.3"
+immich:
+  persistence:
+    library:
+      existingClaim: immich-storage
+server:
+  ingress:
+    main:
+      enabled: true
+      annogations:
+        ingressClassName: traefik
+        # Set body size to 10G to allow uploading large things
+        traefik.ingress.kubernetes.io/buffering: |
+          maxrequestbodybytes: 10000000000
+          memrequestbodybytes: 20000000000
+      hosts:
+        - host: immich.cluster
+          paths:
+            - path: "/"
+machine-learning:
+  persistence:
+    cache:
+      type: pvc
+      storageClass: longhorn-default
+      size: 25Gi

manifests/kustomization.yaml

@@ -1,8 +1,10 @@
 resources:
   - namespaces
+  - pvc
   - helm
   - bitwarden
   - secrets
   - databases
   - matrix
   - gitlab-runner
+  - immich

manifests/pvc/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - immich.yaml

manifests/secrets/kustomization.yaml

@@ -2,6 +2,7 @@ resources:
   - postgres-user-gitlab.yaml
   - postgres-user-pgadmin.yaml
   - postgres-user-matrix.yaml
+  - postgres-user-immich.yaml
   - k3sbackup.yaml
   - gitlab-runner.yaml
   - longhorn.yaml

manifests/values/cnpg.yaml

@@ -0,0 +1,3 @@
+crds:
+  create: true
+includeCRDs: true