Enable Longhorn at last

Enable the iscsi services on the local hosts
Enable special NixOS style Longhorn workarounds
Enable Longhorn in Kubernetes, as well
Update host names to point at Longhorn
Expose Longhorn UI and secrets

manifests/apply.sh

@@ -6,9 +6,26 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 # Change to the script directory
 cd "$SCRIPT_DIR"
 
+# First, label the nodes to control Longhorn rollout
+for n in isaiah jeremiah zeke; do
+    kubectl label nodes "${n}" "node.longhorn.io/create-default-disk=config"
+done
+# Now, configure longhorn settings for each node
+kubectl annotate nodes isaiah 'node.longhorn.io/default-disks-config=[
+  { "path": "/var/lib/longhorn", "allowScheduling" : true }
+]'
+kubectl annotate nodes jeremiah 'node.longhorn.io/default-disks-config=[
+  { "path": "/var/lib/longhorn", "allowScheduling" : true }
+]'
+kubectl annotate nodes zeke 'node.longhorn.io/default-disks-config=[
+  { "path": "/var/lib/longhorn", "allowScheduling" : true }
+]'
+
 kubectl apply -k namespaces
 kubectl apply -f helm/flux.yaml
 sleep 5
+kubectl apply -f helm/kyverno.yaml
+sleep 15
 kubectl apply -k helm
 sleep 5
 kubectl apply -k .

manifests/helm/kustomization.yaml

@@ -1,5 +1,7 @@
 resources:
   - flux.yaml
+  - kyverno.yaml  # Needed to configure Longhorn
+  - longhorn.yaml  # Needed for storage
   - traefik.yaml
   - external-secrets.yaml
   - cloudnative-pg.yaml

manifests/helm/kyverno.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kyverno-system
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: kyverno
+  namespace: kyverno-system
+spec:
+  interval: "24h"
+  url: "https://kyverno.github.io/kyverno/"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kyverno
+  namespace: kyverno-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: kyverno
+      version: "3.4.4"
+      sourceRef:
+        kind: HelmRepository
+        name: kyverno
+      interval: "1h"
+  values:
+    admissionController:
+      replicas: 3
+    backgroundController:
+      replicas: 3
+    cleanupController:
+      replicas: 2
+    reportsController:
+      replicas: 2
+    crds:
+      install: true

manifests/helm/longhorn.yaml

@@ -0,0 +1,115 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: "24h"
+  url: "https://charts.longhorn.io"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: longhorn
+      version: "1.9.0"
+      sourceRef:
+        kind: HelmRepository
+        name: longhorn
+      interval: "1h"
+  values:
+    defaultSettings:
+      createDefaultDiskLabeledNodes: true
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-custom-path
+  namespace: longhorn-system
+data:
+  PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-host-path-to-longhorn
+  annotations:
+    policies.kyverno.io/title: Add Environment Variables from ConfigMap
+    policies.kyverno.io/subject: Pod
+    policies.kyverno.io/category: Other
+    policies.kyverno.io/description: >-
+      Longhorn invokes executables on the host system, and needs
+      to be aware of the host systems PATH. This modifies all
+      deployments such that the PATH is explicitly set to support
+      NixOS based systems.
+spec:
+  rules:
+    - name: add-env-vars
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaces:
+            - longhorn-system
+      mutate:
+        patchStrategicMerge:
+          spec:
+            initContainers:
+              - (name): "*"
+                envFrom:
+                  - configMapRef:
+                      name: longhorn-custom-path
+            containers:
+              - (name): "*"
+                envFrom:
+                  - configMapRef:
+                      name: longhorn-custom-path
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  namespace: longhorn-system
+  name: basic-auth
+spec:
+  basicAuth:
+    realm: Traefik
+    secret: longhorn-ui
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: longhorn-ingress
+  namespace: longhorn-system
+  annotations:
+    ingressClassName: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-basic-auth@kubernetescrd
+    # Set body size to 10G to allow uploading large things
+    traefik.ingress.kubernetes.io/buffering: |
+      maxrequestbodybytes: 10000000000
+      memrequestbodybytes: 20000000000
+spec:
+  ingressClassName: traefik
+  rules:
+  - &host
+    host: longhorn.cluster
+    http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: longhorn-frontend
+            port:
+              number: 80
+  - <<: *host
+    host: longhorn.kubernetes

manifests/secrets/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - postgres-user-matrix.yaml
   - k3sbackup.yaml
   - gitlab-runner.yaml
+  - longhorn.yaml

manifests/secrets/longhorn.yaml

@@ -0,0 +1,33 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: longhorn-ui
+  namespace: longhorn-system
+spec:
+  target:
+    name: longhorn-ui
+    deletionPolicy: Delete
+    template:
+      type: kubernetes.io/basic-auth
+      data:
+        username: |-
+          {{ .username }}
+        password: |-
+          {{ .password }}
+  data:
+    - secretKey: username
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: cbf2bf72-c129-437a-8a75-b30f005d29ec
+        property: username
+    - secretKey: password
+      sourceRef:
+        storeRef:
+          name: bitwarden-login
+          kind: ClusterSecretStore
+      remoteRef:
+        key: cbf2bf72-c129-437a-8a75-b30f005d29ec
+        property: password