greg/nixos
1
0
Fork 1
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Enable Longhorn at last

Browse Source 
Enable the iscsi services on the local hosts
Enable special NixOS style Longhorn workarounds
Enable Longhorn in Kubernetes, as well
Update host names to point at Longhorn
Expose Longhorn UI and secrets
This commit is contained in:
Greg Hellings
2025-07-03 01:36:59 -05:00
parent 9940e89d2e
commit c7e96d1f29
10 changed files with 226 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/genesis/net/hosts
+2 -2
View File
@@ -24,7 +24,7 @@
10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan
# VIP
10.42.5.1 pgadmin.cluter postgres.cluster matrix.cluster
10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster
# IPMI
10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan
@@ -36,7 +36,7 @@
100.88.91.27 genesis.home smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home
100.91.131.66 gitlab.home gitlab.shire-zebra.ts.net gitlab.thehellings.lan registry.thehellings.lan git.thehellings.lan
100.68.203.1 hosea.home hosea.shire-zebra.ts.net
100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes
100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes longhorn.kubernetes
100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes
100.90.74.19 zeke.home
100.115.57.8 linode.home
hosts/isaiah/hardware-configuration.nix
+5
View File
@@ -46,6 +46,11 @@
fsType = "btrfs";
};
fileSystems."/var/lib/longhorn" = {
device = "/dev/disk/by-uuid/b9f5ace7-d224-4aff-8770-d5a9d22be2ae";
fsType = "xfs";
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
hosts/jeremiah/hardware-configuration.nix
+5
View File
@@ -33,6 +33,11 @@
fsType = "vfat";
};
fileSystems."/var/lib/longhorn" = {
device = "/dev/disk/by-uuid/1c896717-4a01-4136-825d-6c0160a78256";
fsType = "xfs";
};
swapDevices = [ ];
# Enables DHCP on each ethernet and wireless interface. In case of scripted networking
manifests/apply.sh
+17
View File
@@ -6,9 +6,26 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
# Change to the script directory
cd "$SCRIPT_DIR"
# First, label the nodes to control Longhorn rollout
for n in isaiah jeremiah zeke; do
kubectl label nodes "${n}" "node.longhorn.io/create-default-disk=config"
done
# Now, configure longhorn settings for each node
kubectl annotate nodes isaiah 'node.longhorn.io/default-disks-config=[
{ "path": "/var/lib/longhorn", "allowScheduling" : true }
]'
kubectl annotate nodes jeremiah 'node.longhorn.io/default-disks-config=[
{ "path": "/var/lib/longhorn", "allowScheduling" : true }
]'
kubectl annotate nodes zeke 'node.longhorn.io/default-disks-config=[
{ "path": "/var/lib/longhorn", "allowScheduling" : true }
]'
kubectl apply -k namespaces
kubectl apply -f helm/flux.yaml
sleep 5
kubectl apply -f helm/kyverno.yaml
sleep 15
kubectl apply -k helm
sleep 5
kubectl apply -k .
manifests/helm/kustomization.yaml
+2
View File
@@ -1,5 +1,7 @@
resources:
- flux.yaml
- kyverno.yaml # Needed to configure Longhorn
- longhorn.yaml # Needed for storage
- traefik.yaml
- external-secrets.yaml
- cloudnative-pg.yaml
manifests/helm/kyverno.yaml
+40
View File
@@ -0,0 +1,40 @@
apiVersion: v1
kind: Namespace
metadata:
name: kyverno-system
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: kyverno
namespace: kyverno-system
spec:
interval: "24h"
url: "https://kyverno.github.io/kyverno/"
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: kyverno
namespace: kyverno-system
spec:
interval: 10m
chart:
spec:
chart: kyverno
version: "3.4.4"
sourceRef:
kind: HelmRepository
name: kyverno
interval: "1h"
values:
admissionController:
replicas: 3
backgroundController:
replicas: 3
cleanupController:
replicas: 2
reportsController:
replicas: 2
crds:
install: true
manifests/helm/longhorn.yaml
+115
View File
@@ -0,0 +1,115 @@
apiVersion: v1
kind: Namespace
metadata:
name: longhorn-system
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: longhorn
namespace: longhorn-system
spec:
interval: "24h"
url: "https://charts.longhorn.io"
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: longhorn
namespace: longhorn-system
spec:
interval: 10m
chart:
spec:
chart: longhorn
version: "1.9.0"
sourceRef:
kind: HelmRepository
name: longhorn
interval: "1h"
values:
defaultSettings:
createDefaultDiskLabeledNodes: true
---
apiVersion: v1
kind: ConfigMap
metadata:
name: longhorn-custom-path
namespace: longhorn-system
data:
PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin
---
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: add-host-path-to-longhorn
annotations:
policies.kyverno.io/title: Add Environment Variables from ConfigMap
policies.kyverno.io/subject: Pod
policies.kyverno.io/category: Other
policies.kyverno.io/description: >-
Longhorn invokes executables on the host system, and needs
to be aware of the host systems PATH. This modifies all
deployments such that the PATH is explicitly set to support
NixOS based systems.
spec:
rules:
- name: add-env-vars
match:
resources:
kinds:
- Pod
namespaces:
- longhorn-system
mutate:
patchStrategicMerge:
spec:
initContainers:
- (name): "*"
envFrom:
- configMapRef:
name: longhorn-custom-path
containers:
- (name): "*"
envFrom:
- configMapRef:
name: longhorn-custom-path
---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
namespace: longhorn-system
name: basic-auth
spec:
basicAuth:
realm: Traefik
secret: longhorn-ui
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: longhorn-ingress
namespace: longhorn-system
annotations:
ingressClassName: traefik
traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-basic-auth@kubernetescrd
# Set body size to 10G to allow uploading large things
traefik.ingress.kubernetes.io/buffering: |
maxrequestbodybytes: 10000000000
memrequestbodybytes: 20000000000
spec:
ingressClassName: traefik
rules:
- &host
host: longhorn.cluster
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: longhorn-frontend
port:
number: 80
- <<: *host
host: longhorn.kubernetes
manifests/secrets/kustomization.yaml
+1
View File
@@ -4,3 +4,4 @@ resources:
- postgres-user-matrix.yaml
- k3sbackup.yaml
- gitlab-runner.yaml
- longhorn.yaml
manifests/secrets/longhorn.yaml
+33
View File
@@ -0,0 +1,33 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: longhorn-ui
namespace: longhorn-system
spec:
target:
name: longhorn-ui
deletionPolicy: Delete
template:
type: kubernetes.io/basic-auth
data:
username: |-
{{ .username }}
password: |-
{{ .password }}
data:
- secretKey: username
sourceRef:
storeRef:
name: bitwarden-login
kind: ClusterSecretStore
remoteRef:
key: cbf2bf72-c129-437a-8a75-b30f005d29ec
property: username
- secretKey: password
sourceRef:
storeRef:
name: bitwarden-login
kind: ClusterSecretStore
remoteRef:
key: cbf2bf72-c129-437a-8a75-b30f005d29ec
property: password
modules/nixos/kubernetes.nix
+6
View File
@@ -43,6 +43,8 @@ in
pkgs.kubernetes-helm
pkgs.kustomize
pkgs.k9s
pkgs.openiscsi
pkgs.nfs-utils # Needed for Longhorn
];
networking.firewall = {
@@ -111,6 +113,10 @@ in
'';
};
};
openiscsi = {
enable = true;
name = "${config.networking.hostName}-initiatorhost";
};
};
users = {