Install Helm as objects

Rather than running helm on the client and passing it to kubectl, just
install Flux and set the Helm repos on the server side

Add resolv.conf to the Kubernetes config so that the hosts can resolve
hostnames in order to pull down the Helm repositories

Add backups to the database configurations

manifests/apply.sh

@@ -7,10 +7,8 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 cd "$SCRIPT_DIR"
 
 kubectl apply -k namespaces
-kustomize build external-secrets --enable-helm | kubectl apply -f -
+kubectl apply -f helm/flux.yaml
-echo -e "\nPausing for a moment to allow the External Secrets stuff to get going\n"
 sleep 5
-kustomize build postgres --enable-helm | kubectl apply -f - --server-side --force-conflicts
+kubectl apply -k helm
-echo -e "\nPausing for a moment to allow Postgres stuff to get going\n"
 sleep 5
-kubectl apply -k .
+kubectl apply .

manifests/databases/postgres-cluster.yaml

@@ -31,3 +31,28 @@ spec:
         superuser: false
         passwordSecret:
           name: postgres-user-matrix
+  backup:
+    retentionPolicy: "30d"
+    barmanObjectStore:
+      destinationPath: "s3://k3sbackup/postgres"
+      endpointURL: "http://s3.thehellings.lan:9000/"
+      s3Credentials:
+        accessKeyId:
+          name: k3sbackup
+          key: username
+        secretAccessKey:
+          name: k3sbackup
+          key: password
+      wal:
+        compression: gzip
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: postgres-backup
+spec:
+  immediate: true # Create one when this is added to the cluster
+  schedule: "0 0 0 * * *" # Midnight, nightly
+  backupOwnerReference: self
+  cluster:
+    name: postgres

manifests/external-secrets/kustomization.yaml

@@ -1,13 +0,0 @@
-namespace: external-secrets
-
-helmCharts:
-  # Install the operator first (with CRDs)
-  - name: external-secrets
-    repo: "https://charts.external-secrets.io/"
-    version: "0.17.0"
-    namespace: external-secrets
-    releaseName: external-secrets
-    includeCRDs: true
-    valuesInline:
-      crds:
-        create: true

manifests/helm/cloudnative-pg.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cnpg-system
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: cloudnative-pg
+  namespace: cnpg-system
+spec:
+  interval: "24h"
+  url: "https://cloudnative-pg.github.io/charts/"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cnpg
+  namespace: cnpg-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: cloudnative-pg
+      version: "0.23.2"
+      sourceRef:
+        kind: HelmRepository
+        name: cloudnative-pg
+      interval: "1h"
+  values:
+    crds:
+      create: true
+    includeCRDs: true

manifests/helm/external-secrets.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: external-secrets
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: external-secrets
+  namespace: external-secrets
+spec:
+  interval: "24h"
+  url: "https://charts.external-secrets.io/"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: external-secrets
+  namespace: external-secrets
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: external-secrets
+      version: "0.17.0"
+      sourceRef:
+        kind: HelmRepository
+        name: external-secrets
+      interval: "1h"
+  values:
+    crds:
+      create: true
+    includeCRDs: true

manifests/helm/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - flux.yaml
+  - external-secrets.yaml
+  - cloudnative-pg.yaml

manifests/kustomization.yaml

@@ -1,5 +1,6 @@
 resources:
   - namespaces
+  - helm
   - bitwarden
   - secrets
   - databases

manifests/namespaces/external-secrets.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: external-secrets

manifests/namespaces/kustomization.yaml

@@ -1,5 +1,4 @@
 resources:
-  - external-secrets.yaml
   - bitwarden.yaml
   - db.yaml
   - matrix.yaml

manifests/postgres/kustomization.yaml

@@ -1,13 +0,0 @@
-namespace: db
-
-helmCharts:
-  # Install the operator first (with CRDs)
-  - name: cloudnative-pg
-    repo: "https://cloudnative-pg.github.io/charts/"
-    version: "0.23.2"
-    namespace: cnpg-system
-    releaseName: cnpg
-    includeCRDs: true
-    valuesInline:
-      crds:
-        create: true

manifests/secrets/k3sbackup.yaml

@@ -0,0 +1,33 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: k3sbackup
+  namespace: db
+spec:
+  target:
+    name: k3sbackup
+    deletionPolicy: Delete
+    template:
+      type: Opaque
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 685b29c6-9264-4e60-ba4a-b2ea005a5d7b
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 685b29c6-9264-4e60-ba4a-b2ea005a5d7b
+        property: password

manifests/secrets/kustomization.yaml

@@ -2,3 +2,4 @@ resources:
   - postgres-user-gitlab.yaml
   - postgres-user-pgadmin.yaml
   - postgres-user-matrix.yaml
+  - k3sbackup.yaml

modules/nixos/kubernetes.nix

@@ -27,6 +27,7 @@ in
     };
 
     environment.systemPackages = [
+      pkgs.fluxcd
       pkgs.kubectl-cnpg
       pkgs.kubernetes-helm
       pkgs.kustomize
@@ -59,6 +60,7 @@ in
         "--service-cidr=10.221.0.0/16"
         "--write-kubeconfig-mode 0640"
         "--write-kubeconfig-group kubeconfig"
+        "--resolv-conf=/etc/resolv.conf"
         "--tls-san ${config.networking.hostName}.home"
         "--tls-san ${config.networking.hostName}.thehellings.lan"
         "--tls-san ${config.networking.hostName}.shire-zebra.ts.net"