From c7e96d1f29d75554e4c6d04aa5452a8e4887d91d Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Thu, 3 Jul 2025 01:36:59 -0500 Subject: [PATCH] Enable Longhorn at last Enable the iscsi services on the local hosts Enable special NixOS style Longhorn workarounds Enable Longhorn in Kubernetes, as well Update host names to point at Longhorn Expose Longhorn UI and secrets --- hosts/genesis/net/hosts | 4 +- hosts/isaiah/hardware-configuration.nix | 5 + hosts/jeremiah/hardware-configuration.nix | 5 + manifests/apply.sh | 17 ++++ manifests/helm/kustomization.yaml | 2 + manifests/helm/kyverno.yaml | 40 ++++++++ manifests/helm/longhorn.yaml | 115 ++++++++++++++++++++++ manifests/secrets/kustomization.yaml | 1 + manifests/secrets/longhorn.yaml | 33 +++++++ modules/nixos/kubernetes.nix | 6 ++ 10 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 manifests/helm/kyverno.yaml create mode 100644 manifests/helm/longhorn.yaml create mode 100644 manifests/secrets/longhorn.yaml diff --git a/hosts/genesis/net/hosts b/hosts/genesis/net/hosts index 5228fd0..157a86c 100644 --- a/hosts/genesis/net/hosts +++ b/hosts/genesis/net/hosts @@ -24,7 +24,7 @@ 10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan # VIP -10.42.5.1 pgadmin.cluter postgres.cluster matrix.cluster +10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster # IPMI 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan @@ -36,7 +36,7 @@ 100.88.91.27 genesis.home smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home 100.91.131.66 gitlab.home gitlab.shire-zebra.ts.net gitlab.thehellings.lan registry.thehellings.lan git.thehellings.lan 100.68.203.1 hosea.home hosea.shire-zebra.ts.net -100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes +100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes longhorn.kubernetes 100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes 100.90.74.19 zeke.home 100.115.57.8 linode.home diff --git a/hosts/isaiah/hardware-configuration.nix b/hosts/isaiah/hardware-configuration.nix index 58cdcef..9c6d38e 100644 --- a/hosts/isaiah/hardware-configuration.nix +++ b/hosts/isaiah/hardware-configuration.nix @@ -46,6 +46,11 @@ fsType = "btrfs"; }; + fileSystems."/var/lib/longhorn" = { + device = "/dev/disk/by-uuid/b9f5ace7-d224-4aff-8770-d5a9d22be2ae"; + fsType = "xfs"; + }; + swapDevices = [ ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/hosts/jeremiah/hardware-configuration.nix b/hosts/jeremiah/hardware-configuration.nix index c0c7635..e67f5ce 100644 --- a/hosts/jeremiah/hardware-configuration.nix +++ b/hosts/jeremiah/hardware-configuration.nix @@ -33,6 +33,11 @@ fsType = "vfat"; }; + fileSystems."/var/lib/longhorn" = { + device = "/dev/disk/by-uuid/1c896717-4a01-4136-825d-6c0160a78256"; + fsType = "xfs"; + }; + swapDevices = [ ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/manifests/apply.sh b/manifests/apply.sh index 9ef3d4c8..af84718 100755 --- a/manifests/apply.sh +++ b/manifests/apply.sh @@ -6,9 +6,26 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Change to the script directory cd "$SCRIPT_DIR" +# First, label the nodes to control Longhorn rollout +for n in isaiah jeremiah zeke; do + kubectl label nodes "${n}" "node.longhorn.io/create-default-disk=config" +done +# Now, configure longhorn settings for each node +kubectl annotate nodes isaiah 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true } +]' +kubectl annotate nodes jeremiah 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true } +]' +kubectl annotate nodes zeke 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true } +]' + kubectl apply -k namespaces kubectl apply -f helm/flux.yaml sleep 5 +kubectl apply -f helm/kyverno.yaml +sleep 15 kubectl apply -k helm sleep 5 kubectl apply -k . diff --git a/manifests/helm/kustomization.yaml b/manifests/helm/kustomization.yaml index c6ae679..5f37ea3 100644 --- a/manifests/helm/kustomization.yaml +++ b/manifests/helm/kustomization.yaml @@ -1,5 +1,7 @@ resources: - flux.yaml + - kyverno.yaml # Needed to configure Longhorn + - longhorn.yaml # Needed for storage - traefik.yaml - external-secrets.yaml - cloudnative-pg.yaml diff --git a/manifests/helm/kyverno.yaml b/manifests/helm/kyverno.yaml new file mode 100644 index 0000000..bce3318 --- /dev/null +++ b/manifests/helm/kyverno.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kyverno-system +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: kyverno + namespace: kyverno-system +spec: + interval: "24h" + url: "https://kyverno.github.io/kyverno/" +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kyverno + namespace: kyverno-system +spec: + interval: 10m + chart: + spec: + chart: kyverno + version: "3.4.4" + sourceRef: + kind: HelmRepository + name: kyverno + interval: "1h" + values: + admissionController: + replicas: 3 + backgroundController: + replicas: 3 + cleanupController: + replicas: 2 + reportsController: + replicas: 2 + crds: + install: true diff --git a/manifests/helm/longhorn.yaml b/manifests/helm/longhorn.yaml new file mode 100644 index 0000000..8b0bf12 --- /dev/null +++ b/manifests/helm/longhorn.yaml @@ -0,0 +1,115 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: longhorn + namespace: longhorn-system +spec: + interval: "24h" + url: "https://charts.longhorn.io" +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: longhorn + namespace: longhorn-system +spec: + interval: 10m + chart: + spec: + chart: longhorn + version: "1.9.0" + sourceRef: + kind: HelmRepository + name: longhorn + interval: "1h" + values: + defaultSettings: + createDefaultDiskLabeledNodes: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-custom-path + namespace: longhorn-system +data: + PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-host-path-to-longhorn + annotations: + policies.kyverno.io/title: Add Environment Variables from ConfigMap + policies.kyverno.io/subject: Pod + policies.kyverno.io/category: Other + policies.kyverno.io/description: >- + Longhorn invokes executables on the host system, and needs + to be aware of the host systems PATH. This modifies all + deployments such that the PATH is explicitly set to support + NixOS based systems. +spec: + rules: + - name: add-env-vars + match: + resources: + kinds: + - Pod + namespaces: + - longhorn-system + mutate: + patchStrategicMerge: + spec: + initContainers: + - (name): "*" + envFrom: + - configMapRef: + name: longhorn-custom-path + containers: + - (name): "*" + envFrom: + - configMapRef: + name: longhorn-custom-path +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + namespace: longhorn-system + name: basic-auth +spec: + basicAuth: + realm: Traefik + secret: longhorn-ui +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: longhorn-ingress + namespace: longhorn-system + annotations: + ingressClassName: traefik + traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-basic-auth@kubernetescrd + # Set body size to 10G to allow uploading large things + traefik.ingress.kubernetes.io/buffering: | + maxrequestbodybytes: 10000000000 + memrequestbodybytes: 20000000000 +spec: + ingressClassName: traefik + rules: + - &host + host: longhorn.cluster + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: longhorn-frontend + port: + number: 80 + - <<: *host + host: longhorn.kubernetes diff --git a/manifests/secrets/kustomization.yaml b/manifests/secrets/kustomization.yaml index e132e9f..ba8494d 100644 --- a/manifests/secrets/kustomization.yaml +++ b/manifests/secrets/kustomization.yaml @@ -4,3 +4,4 @@ resources: - postgres-user-matrix.yaml - k3sbackup.yaml - gitlab-runner.yaml + - longhorn.yaml diff --git a/manifests/secrets/longhorn.yaml b/manifests/secrets/longhorn.yaml new file mode 100644 index 0000000..28d7402 --- /dev/null +++ b/manifests/secrets/longhorn.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: longhorn-ui + namespace: longhorn-system +spec: + target: + name: longhorn-ui + deletionPolicy: Delete + template: + type: kubernetes.io/basic-auth + data: + username: |- + {{ .username }} + password: |- + {{ .password }} + data: + - secretKey: username + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: cbf2bf72-c129-437a-8a75-b30f005d29ec + property: username + - secretKey: password + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: cbf2bf72-c129-437a-8a75-b30f005d29ec + property: password diff --git a/modules/nixos/kubernetes.nix b/modules/nixos/kubernetes.nix index cd0b647..d88a859 100644 --- a/modules/nixos/kubernetes.nix +++ b/modules/nixos/kubernetes.nix @@ -43,6 +43,8 @@ in pkgs.kubernetes-helm pkgs.kustomize pkgs.k9s + pkgs.openiscsi + pkgs.nfs-utils # Needed for Longhorn ]; networking.firewall = { @@ -111,6 +113,10 @@ in ''; }; }; + openiscsi = { + enable = true; + name = "${config.networking.hostName}-initiatorhost"; + }; }; users = {