Enable Longhorn at last

Enable the iscsi services on the local hosts
Enable special NixOS style Longhorn workarounds
Enable Longhorn in Kubernetes, as well
Update host names to point at Longhorn
Expose Longhorn UI and secrets

manifests/helm/kustomization.yaml

@@ -1,5 +1,7 @@
 resources:
   - flux.yaml
+  - kyverno.yaml  # Needed to configure Longhorn
+  - longhorn.yaml  # Needed for storage
   - traefik.yaml
   - external-secrets.yaml
   - cloudnative-pg.yaml

manifests/helm/kyverno.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kyverno-system
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: kyverno
+  namespace: kyverno-system
+spec:
+  interval: "24h"
+  url: "https://kyverno.github.io/kyverno/"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kyverno
+  namespace: kyverno-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: kyverno
+      version: "3.4.4"
+      sourceRef:
+        kind: HelmRepository
+        name: kyverno
+      interval: "1h"
+  values:
+    admissionController:
+      replicas: 3
+    backgroundController:
+      replicas: 3
+    cleanupController:
+      replicas: 2
+    reportsController:
+      replicas: 2
+    crds:
+      install: true

manifests/helm/longhorn.yaml

@@ -0,0 +1,115 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: "24h"
+  url: "https://charts.longhorn.io"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: longhorn
+      version: "1.9.0"
+      sourceRef:
+        kind: HelmRepository
+        name: longhorn
+      interval: "1h"
+  values:
+    defaultSettings:
+      createDefaultDiskLabeledNodes: true
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-custom-path
+  namespace: longhorn-system
+data:
+  PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-host-path-to-longhorn
+  annotations:
+    policies.kyverno.io/title: Add Environment Variables from ConfigMap
+    policies.kyverno.io/subject: Pod
+    policies.kyverno.io/category: Other
+    policies.kyverno.io/description: >-
+      Longhorn invokes executables on the host system, and needs
+      to be aware of the host systems PATH. This modifies all
+      deployments such that the PATH is explicitly set to support
+      NixOS based systems.
+spec:
+  rules:
+    - name: add-env-vars
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaces:
+            - longhorn-system
+      mutate:
+        patchStrategicMerge:
+          spec:
+            initContainers:
+              - (name): "*"
+                envFrom:
+                  - configMapRef:
+                      name: longhorn-custom-path
+            containers:
+              - (name): "*"
+                envFrom:
+                  - configMapRef:
+                      name: longhorn-custom-path
+---
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  namespace: longhorn-system
+  name: basic-auth
+spec:
+  basicAuth:
+    realm: Traefik
+    secret: longhorn-ui
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: longhorn-ingress
+  namespace: longhorn-system
+  annotations:
+    ingressClassName: traefik
+    traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-basic-auth@kubernetescrd
+    # Set body size to 10G to allow uploading large things
+    traefik.ingress.kubernetes.io/buffering: |
+      maxrequestbodybytes: 10000000000
+      memrequestbodybytes: 20000000000
+spec:
+  ingressClassName: traefik
+  rules:
+  - &host
+    host: longhorn.cluster
+    http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: longhorn-frontend
+            port:
+              number: 80
+  - <<: *host
+    host: longhorn.kubernetes