nixos/manifests/helm/longhorn.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: longhorn-system
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
  name: longhorn
  namespace: longhorn-system
spec:
  interval: "24h"
  url: "https://charts.longhorn.io"
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: longhorn
  namespace: longhorn-system
spec:
  interval: 10m
  dependsOn:
    - name: tailscale
      namespace: tailscale
    - name: kyverno
      namespace: kyverno-system
  chart:
    spec:
      chart: longhorn
      version: "1.11.3"
      sourceRef:
        kind: HelmRepository
        name: longhorn
      interval: "1h"
  values:
    defaultSettings:
      createDefaultDiskLabeledNodes: true
      replicaSoftAntiAffinity: true
      replicaAutoBalance: "best-effort"
    persistence:
      # This should go back to 3 when I have more cluster
      defaultClassReplicaCount: 3
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: longhorn-custom-path
  namespace: longhorn-system
data:
  # yamllint disable-line rule:line-length
  PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: add-host-path-to-longhorn
  annotations:
    policies.kyverno.io/title: Add Environment Variables from ConfigMap
    policies.kyverno.io/subject: Pod
    policies.kyverno.io/category: Other
    policies.kyverno.io/description: >-
      Longhorn invokes executables on the host system, and needs
      to be aware of the host systems PATH. This modifies all
      deployments such that the PATH is explicitly set to support
      NixOS based systems.
spec:
  rules:
    - name: add-env-vars
      match:
        resources:
          kinds:
            - Pod
          namespaces:
            - longhorn-system
      mutate:
        patchStrategicMerge:
          spec:
            initContainers:
              - (name): "*"
                envFrom:
                  - configMapRef:
                      name: longhorn-custom-path
            containers:
              - (name): "*"
                envFrom:
                  - configMapRef:
                      name: longhorn-custom-path
---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
  namespace: longhorn-system
  name: basic-auth
spec:
  basicAuth:
    realm: Traefik
    secret: longhorn-ui
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: longhorn-ingress-tailscale
  namespace: longhorn-system
spec:
  ingressClassName: tailscale
  defaultBackend:
    service:
      name: longhorn-frontend
      port:
        number: 80
  tls:
    - hosts:
        - longhorn
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: longhorn-ingress
  namespace: longhorn-system
  annotations:
    ingressClassName: traefik
    # yamllint disable-line rule:line-length
    traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-basic-auth@kubernetescrd
    # Set body size to 10G to allow uploading large things
    traefik.ingress.kubernetes.io/buffering: |
      maxrequestbodybytes: 10000000000
      memrequestbodybytes: 20000000000
spec:
  ingressClassName: traefik
  rules:
    - &host
      host: longhorn.cluster
      http:
        paths:
          - pathType: Prefix
            path: "/"
            backend:
              service:
                name: longhorn-frontend
                port:
                  number: 80
    - <<: *host
      host: longhorn.kubernetes
    - <<: *host
      host: longhorn.k3s.nebula.thehellings.com
    - <<: *host
      host: longhorn.k3s.thehellings.lan
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: longhorn-default
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: Immediate
parameters:
  backupTargetName: default
  numberOfReplicas: "2"
  staleReplicaTimeout: "2880"
  fromBackup: ""
  fsType: ext4
  diskSelector: "hdd,large"
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: longhorn-3
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: Immediate
parameters:
  backupTargetName: default
  numberOfReplicas: "3"
  staleReplicaTimeout: "2880"
  fromBackup: ""
  fsType: ext4
  diskSelector: "hdd,large"