diff --git a/hosts/genesis/net/hosts b/hosts/genesis/net/hosts index 157a86c..00c6b9e 100644 --- a/hosts/genesis/net/hosts +++ b/hosts/genesis/net/hosts @@ -24,7 +24,7 @@ 10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan # VIP -10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster +10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster immich.cluster # IPMI 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan @@ -37,7 +37,7 @@ 100.91.131.66 gitlab.home gitlab.shire-zebra.ts.net gitlab.thehellings.lan registry.thehellings.lan git.thehellings.lan 100.68.203.1 hosea.home hosea.shire-zebra.ts.net 100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes longhorn.kubernetes -100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes +100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes immich.kubernetes 100.90.74.19 zeke.home 100.115.57.8 linode.home 100.65.5.38 matrix.home matrix.shire-zebra.ts.net diff --git a/manifests/apply.sh b/manifests/apply.sh index af84718..21f056a 100755 --- a/manifests/apply.sh +++ b/manifests/apply.sh @@ -11,14 +11,14 @@ for n in isaiah jeremiah zeke; do kubectl label nodes "${n}" "node.longhorn.io/create-default-disk=config" done # Now, configure longhorn settings for each node -kubectl annotate nodes isaiah 'node.longhorn.io/default-disks-config=[ - { "path": "/var/lib/longhorn", "allowScheduling" : true } +kubectl annotate nodes --overwrite isaiah 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true, "tags": ["hdd", "large"]} ]' -kubectl annotate nodes jeremiah 'node.longhorn.io/default-disks-config=[ - { "path": "/var/lib/longhorn", "allowScheduling" : true } +kubectl annotate nodes --overwrite jeremiah 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true, "tags": ["hdd", "large"]} ]' -kubectl annotate nodes zeke 'node.longhorn.io/default-disks-config=[ - { "path": "/var/lib/longhorn", "allowScheduling" : true } +kubectl annotate nodes --overwrite zeke 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : trues, "tags": ["ssd", "fast"]} ]' kubectl apply -k namespaces @@ -29,3 +29,13 @@ sleep 15 kubectl apply -k helm sleep 5 kubectl apply -k . +sleep 5 +# https://cloudnative-pg.io +helm repo add cnpg https://cloudnative-pg.github.io/charts/ +helm upgrade --install cnpg \ + --create-namespace --namespace cnpg-system \ + cnpg/cloudnative-pg \ + -f values/cnpg.yaml \ + --wait + +./immich/apply.sh diff --git a/manifests/helm/cloudnative-pg.yaml b/manifests/helm/cloudnative-pg.yaml deleted file mode 100644 index 46fe07d..0000000 --- a/manifests/helm/cloudnative-pg.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: cnpg-system ---- -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: cloudnative-pg - namespace: cnpg-system -spec: - interval: "24h" - url: "https://cloudnative-pg.github.io/charts/" ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: cnpg - namespace: cnpg-system -spec: - interval: 10m - chart: - spec: - chart: cloudnative-pg - version: "0.23.2" - sourceRef: - kind: HelmRepository - name: cloudnative-pg - interval: "1h" - values: - crds: - create: true - includeCRDs: true diff --git a/manifests/helm/kustomization.yaml b/manifests/helm/kustomization.yaml index 5f37ea3..531baaf 100644 --- a/manifests/helm/kustomization.yaml +++ b/manifests/helm/kustomization.yaml @@ -4,4 +4,3 @@ resources: - longhorn.yaml # Needed for storage - traefik.yaml - external-secrets.yaml - - cloudnative-pg.yaml diff --git a/manifests/helm/longhorn.yaml b/manifests/helm/longhorn.yaml index 8b0bf12..e72e098 100644 --- a/manifests/helm/longhorn.yaml +++ b/manifests/helm/longhorn.yaml @@ -113,3 +113,35 @@ spec: number: 80 - <<: *host host: longhorn.kubernetes +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-default +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + backupTargetName: default + numberOfReplicas: "2" + staleReplicaTimeout: "2880" + fromBackup: "" + fsType: ext4 + diskSelector: "hdd,large" +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-fast +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + backupTargetName: default + numberOfReplicas: "1" + staleReplicaTimeout: "2880" + fromBackup: "" + fsType: ext4 + diskSelector: "ssd,fast" diff --git a/manifests/immich/apply.sh b/manifests/immich/apply.sh new file mode 100755 index 0000000..03213a4 --- /dev/null +++ b/manifests/immich/apply.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + + +# Get the directory where the script is located +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Change to the script directory +cd "$SCRIPT_DIR" + +kubectl apply -k "$SCRIPT_DIR" + +# https://www.dragonflydb.io/guides/redis-kubernetes +# Deploys into immich namespace, directly, in order to allow the password to be +# accessed by the immich installer +helm upgrade --install --create-namespace --namespace immich redis \ + oci://registry-1.docker.io/bitnamicharts/redis \ + -f "${SCRIPT_DIR}/values-redis.yaml" \ + --wait +# https://github.com/immich-app/immich-charts/tree/main +helm upgrade --install --create-namespace --namespace immich immich \ + oci://ghcr.io/immich-app/immich-charts/immich \ + -f "${SCRIPT_DIR}/values.yaml" \ + --wait diff --git a/manifests/immich/database.yaml b/manifests/immich/database.yaml new file mode 100644 index 0000000..46c0bed --- /dev/null +++ b/manifests/immich/database.yaml @@ -0,0 +1,74 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + namespace: db + name: pgvector +spec: + imageName: "ghcr.io/tensorchord/cloudnative-pgvecto.rs:16-v0.3.0" + instances: 1 + storage: + size: 10Gi + primaryUpdateStrategy: unsupervised + postgresql: + shared_preload_libraries: + - "vectors.so" + + bootstrap: + initdb: + database: immich + owner: immich + secret: + name: postgres-user-immich + dataChecksums: true + postInitApplicationSQL: + - ALTER SYSTEM SET search_path TO "$user", public, vectors; + - SET search_path TO "$user", public, vectors; + - CREATE EXTENSION IF NOT EXISTS "vectors"; + - CREATE EXTENSION IF NOT EXISTS "cube"; + - CREATE EXTENSION IF NOT EXISTS "earthdistance"; + - ALTER SCHEMA vectors OWNER TO "immich"; + - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA vectors TO "immich"; + - GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "immich"; + managed: + roles: + - name: immich + ensure: present + comment: Immich DB user + login: true + superuser: false + passwordSecret: + name: postgres-user-immich + backup: + retentionPolicy: "30d" + barmanObjectStore: + destinationPath: "s3://k3sbackup/pgvector" + endpointURL: "http://s3.thehellings.lan:9000/" + s3Credentials: + accessKeyId: + name: k3sbackup + key: username + secretAccessKey: + name: k3sbackup + key: password + wal: + compression: gzip +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + namespace: db + name: database-immich +spec: + name: immich + owner: immich + cluster: + name: pgvector + extensions: + - name: vectors + ensure: present + - name: vectorchord + ensure: present + - name: cube + ensure: present + - name: earthdistance + ensure: present diff --git a/manifests/immich/ingress.yaml b/manifests/immich/ingress.yaml new file mode 100644 index 0000000..9ccb756 --- /dev/null +++ b/manifests/immich/ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + namespace: immich + name: immich + annotations: + ingressClassName: traefik + # Set body size to 10G to allow uploading large things + traefik.ingress.kubernetes.io/buffering: | + maxrequestbodybytes: 10000000000 + memrequestbodybytes: 20000000000 +spec: + rules: + - &host + host: immich.kubernetes + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: immich-server + port: + name: http + - <<: *host + host: immich.thehellings.com diff --git a/manifests/immich/kustomization.yaml b/manifests/immich/kustomization.yaml new file mode 100644 index 0000000..db99467 --- /dev/null +++ b/manifests/immich/kustomization.yaml @@ -0,0 +1,6 @@ +resources: + - namespace.yaml + - postgres-user-secret.yaml + - database.yaml + - pvc.yaml + - ingress.yaml diff --git a/manifests/immich/namespace.yaml b/manifests/immich/namespace.yaml new file mode 100644 index 0000000..c796392 --- /dev/null +++ b/manifests/immich/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: immich diff --git a/manifests/immich/postgres-user-secret.yaml b/manifests/immich/postgres-user-secret.yaml new file mode 100644 index 0000000..1e100bf --- /dev/null +++ b/manifests/immich/postgres-user-secret.yaml @@ -0,0 +1,67 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgres-user-immich + namespace: db +spec: + target: + name: postgres-user-immich + deletionPolicy: Delete + template: + type: Opaque + data: + username: |- + {{ .username }} + password: |- + {{ .password }} + data: + - secretKey: username + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f + property: username + - secretKey: password + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f + property: password +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgres-user-immich + namespace: immich +spec: + target: + name: postgres-user-immich + deletionPolicy: Delete + template: + type: Opaque + data: + username: |- + {{ .username }} + password: |- + {{ .password }} + data: + - secretKey: username + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f + property: username + - secretKey: password + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: 5ce63e26-dd7f-46d4-b7ed-b310002ef93f + property: password diff --git a/manifests/immich/pvc.yaml b/manifests/immich/pvc.yaml new file mode 100644 index 0000000..76563de --- /dev/null +++ b/manifests/immich/pvc.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + namespace: immich + name: immich-storage +spec: + storageClassName: longhorn-default + volumeName: immich-storage + resources: + requests: + storage: 250Gi + accessModes: + - ReadWriteOnce +--- +#apiVersion: v1 +#kind: PersistentVolume +#metadata: + #namespace: immich + #name: immich-storage +#spec: + #storageClassName: longhorn-default + #capacity: + #storage: 250Gi + #volumeMode: Filesystem + #accessModes: + #- ReadWriteOnce + #csi: + #driver: driver.longhorn.io + #volumeHandle: immich-storage diff --git a/manifests/immich/values-redis.yaml b/manifests/immich/values-redis.yaml new file mode 100644 index 0000000..4991c5d --- /dev/null +++ b/manifests/immich/values-redis.yaml @@ -0,0 +1,8 @@ +# https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml +# https://github.com/bitnami/charts/tree/main/bitnami/redis +architecture: standalone +global: + defaultStorageClass: longhorn-default +master: + persistence: + storageClass: longhorn-default diff --git a/manifests/immich/values.yaml b/manifests/immich/values.yaml new file mode 100644 index 0000000..64142bb --- /dev/null +++ b/manifests/immich/values.yaml @@ -0,0 +1,46 @@ +# https://github.com/immich-app/immich-charts/blob/main/charts/immich/values.yaml +env: + DB_HOSTNAME: pgvector-rw.db.svc.cluster.local + DB_DATABASE_NAME: immich + DB_USERNAME: + valueFrom: + secretKeyRef: + name: postgres-user-immich + key: username + DB_PASSWORD: + valueFrom: + secretKeyRef: + name: postgres-user-immich + key: password + REDIS_HOSTNAME: redis-master + REDIS_PASSWORD: + valueFrom: + secretKeyRef: + name: redis + key: redis-password +image: + tag: "v1.135.3" +immich: + persistence: + library: + existingClaim: immich-storage +server: + ingress: + main: + enabled: true + annogations: + ingressClassName: traefik + # Set body size to 10G to allow uploading large things + traefik.ingress.kubernetes.io/buffering: | + maxrequestbodybytes: 10000000000 + memrequestbodybytes: 20000000000 + hosts: + - host: immich.cluster + paths: + - path: "/" +machine-learning: + persistence: + cache: + type: pvc + storageClass: longhorn-default + size: 25Gi diff --git a/manifests/kustomization.yaml b/manifests/kustomization.yaml index d30f511..7f1876f 100644 --- a/manifests/kustomization.yaml +++ b/manifests/kustomization.yaml @@ -1,8 +1,10 @@ resources: - namespaces + - pvc - helm - bitwarden - secrets - databases - matrix - gitlab-runner + - immich diff --git a/manifests/pvc/kustomization.yaml b/manifests/pvc/kustomization.yaml new file mode 100644 index 0000000..2a5138b --- /dev/null +++ b/manifests/pvc/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - immich.yaml diff --git a/manifests/secrets/kustomization.yaml b/manifests/secrets/kustomization.yaml index ba8494d..502d47e 100644 --- a/manifests/secrets/kustomization.yaml +++ b/manifests/secrets/kustomization.yaml @@ -2,6 +2,7 @@ resources: - postgres-user-gitlab.yaml - postgres-user-pgadmin.yaml - postgres-user-matrix.yaml + - postgres-user-immich.yaml - k3sbackup.yaml - gitlab-runner.yaml - longhorn.yaml diff --git a/manifests/values/cnpg.yaml b/manifests/values/cnpg.yaml new file mode 100644 index 0000000..991e377 --- /dev/null +++ b/manifests/values/cnpg.yaml @@ -0,0 +1,3 @@ +crds: + create: true +includeCRDs: true