diff --git a/hosts/genesis/net/hosts b/hosts/genesis/net/hosts index 5228fd0..157a86c 100644 --- a/hosts/genesis/net/hosts +++ b/hosts/genesis/net/hosts @@ -24,7 +24,7 @@ 10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan # VIP -10.42.5.1 pgadmin.cluter postgres.cluster matrix.cluster +10.42.5.1 longhorn.cluster matrix.cluster pgadmin.cluter postgres.cluster # IPMI 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan @@ -36,7 +36,7 @@ 100.88.91.27 genesis.home smart.home zwave.home nixcache.home gitcache.home dashy.home uptime.home speed.home 100.91.131.66 gitlab.home gitlab.shire-zebra.ts.net gitlab.thehellings.lan registry.thehellings.lan git.thehellings.lan 100.68.203.1 hosea.home hosea.shire-zebra.ts.net -100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes +100.84.183.79 isaiah.home isaiah.shire-zebra.ts.net pgadmin.kubernetes postgres.kubernetes longhorn.kubernetes 100.102.186.39 jeremiah.home jeremiah.shire-zebra.ts.net matrix.kubernetes 100.90.74.19 zeke.home 100.115.57.8 linode.home diff --git a/hosts/isaiah/hardware-configuration.nix b/hosts/isaiah/hardware-configuration.nix index 58cdcef..9c6d38e 100644 --- a/hosts/isaiah/hardware-configuration.nix +++ b/hosts/isaiah/hardware-configuration.nix @@ -46,6 +46,11 @@ fsType = "btrfs"; }; + fileSystems."/var/lib/longhorn" = { + device = "/dev/disk/by-uuid/b9f5ace7-d224-4aff-8770-d5a9d22be2ae"; + fsType = "xfs"; + }; + swapDevices = [ ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/hosts/jeremiah/hardware-configuration.nix b/hosts/jeremiah/hardware-configuration.nix index c0c7635..e67f5ce 100644 --- a/hosts/jeremiah/hardware-configuration.nix +++ b/hosts/jeremiah/hardware-configuration.nix @@ -33,6 +33,11 @@ fsType = "vfat"; }; + fileSystems."/var/lib/longhorn" = { + device = "/dev/disk/by-uuid/1c896717-4a01-4136-825d-6c0160a78256"; + fsType = "xfs"; + }; + swapDevices = [ ]; # Enables DHCP on each ethernet and wireless interface. In case of scripted networking diff --git a/manifests/apply.sh b/manifests/apply.sh index 9ef3d4c8..af84718 100755 --- a/manifests/apply.sh +++ b/manifests/apply.sh @@ -6,9 +6,26 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Change to the script directory cd "$SCRIPT_DIR" +# First, label the nodes to control Longhorn rollout +for n in isaiah jeremiah zeke; do + kubectl label nodes "${n}" "node.longhorn.io/create-default-disk=config" +done +# Now, configure longhorn settings for each node +kubectl annotate nodes isaiah 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true } +]' +kubectl annotate nodes jeremiah 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true } +]' +kubectl annotate nodes zeke 'node.longhorn.io/default-disks-config=[ + { "path": "/var/lib/longhorn", "allowScheduling" : true } +]' + kubectl apply -k namespaces kubectl apply -f helm/flux.yaml sleep 5 +kubectl apply -f helm/kyverno.yaml +sleep 15 kubectl apply -k helm sleep 5 kubectl apply -k . diff --git a/manifests/helm/kustomization.yaml b/manifests/helm/kustomization.yaml index c6ae679..5f37ea3 100644 --- a/manifests/helm/kustomization.yaml +++ b/manifests/helm/kustomization.yaml @@ -1,5 +1,7 @@ resources: - flux.yaml + - kyverno.yaml # Needed to configure Longhorn + - longhorn.yaml # Needed for storage - traefik.yaml - external-secrets.yaml - cloudnative-pg.yaml diff --git a/manifests/helm/kyverno.yaml b/manifests/helm/kyverno.yaml new file mode 100644 index 0000000..bce3318 --- /dev/null +++ b/manifests/helm/kyverno.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kyverno-system +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: kyverno + namespace: kyverno-system +spec: + interval: "24h" + url: "https://kyverno.github.io/kyverno/" +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: kyverno + namespace: kyverno-system +spec: + interval: 10m + chart: + spec: + chart: kyverno + version: "3.4.4" + sourceRef: + kind: HelmRepository + name: kyverno + interval: "1h" + values: + admissionController: + replicas: 3 + backgroundController: + replicas: 3 + cleanupController: + replicas: 2 + reportsController: + replicas: 2 + crds: + install: true diff --git a/manifests/helm/longhorn.yaml b/manifests/helm/longhorn.yaml new file mode 100644 index 0000000..8b0bf12 --- /dev/null +++ b/manifests/helm/longhorn.yaml @@ -0,0 +1,115 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-system +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: longhorn + namespace: longhorn-system +spec: + interval: "24h" + url: "https://charts.longhorn.io" +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: longhorn + namespace: longhorn-system +spec: + interval: 10m + chart: + spec: + chart: longhorn + version: "1.9.0" + sourceRef: + kind: HelmRepository + name: longhorn + interval: "1h" + values: + defaultSettings: + createDefaultDiskLabeledNodes: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: longhorn-custom-path + namespace: longhorn-system +data: + PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin +--- +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-host-path-to-longhorn + annotations: + policies.kyverno.io/title: Add Environment Variables from ConfigMap + policies.kyverno.io/subject: Pod + policies.kyverno.io/category: Other + policies.kyverno.io/description: >- + Longhorn invokes executables on the host system, and needs + to be aware of the host systems PATH. This modifies all + deployments such that the PATH is explicitly set to support + NixOS based systems. +spec: + rules: + - name: add-env-vars + match: + resources: + kinds: + - Pod + namespaces: + - longhorn-system + mutate: + patchStrategicMerge: + spec: + initContainers: + - (name): "*" + envFrom: + - configMapRef: + name: longhorn-custom-path + containers: + - (name): "*" + envFrom: + - configMapRef: + name: longhorn-custom-path +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + namespace: longhorn-system + name: basic-auth +spec: + basicAuth: + realm: Traefik + secret: longhorn-ui +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: longhorn-ingress + namespace: longhorn-system + annotations: + ingressClassName: traefik + traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-basic-auth@kubernetescrd + # Set body size to 10G to allow uploading large things + traefik.ingress.kubernetes.io/buffering: | + maxrequestbodybytes: 10000000000 + memrequestbodybytes: 20000000000 +spec: + ingressClassName: traefik + rules: + - &host + host: longhorn.cluster + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: longhorn-frontend + port: + number: 80 + - <<: *host + host: longhorn.kubernetes diff --git a/manifests/secrets/kustomization.yaml b/manifests/secrets/kustomization.yaml index e132e9f..ba8494d 100644 --- a/manifests/secrets/kustomization.yaml +++ b/manifests/secrets/kustomization.yaml @@ -4,3 +4,4 @@ resources: - postgres-user-matrix.yaml - k3sbackup.yaml - gitlab-runner.yaml + - longhorn.yaml diff --git a/manifests/secrets/longhorn.yaml b/manifests/secrets/longhorn.yaml new file mode 100644 index 0000000..28d7402 --- /dev/null +++ b/manifests/secrets/longhorn.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: longhorn-ui + namespace: longhorn-system +spec: + target: + name: longhorn-ui + deletionPolicy: Delete + template: + type: kubernetes.io/basic-auth + data: + username: |- + {{ .username }} + password: |- + {{ .password }} + data: + - secretKey: username + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: cbf2bf72-c129-437a-8a75-b30f005d29ec + property: username + - secretKey: password + sourceRef: + storeRef: + name: bitwarden-login + kind: ClusterSecretStore + remoteRef: + key: cbf2bf72-c129-437a-8a75-b30f005d29ec + property: password diff --git a/modules/nixos/kubernetes.nix b/modules/nixos/kubernetes.nix index cd0b647..d88a859 100644 --- a/modules/nixos/kubernetes.nix +++ b/modules/nixos/kubernetes.nix @@ -43,6 +43,8 @@ in pkgs.kubernetes-helm pkgs.kustomize pkgs.k9s + pkgs.openiscsi + pkgs.nfs-utils # Needed for Longhorn ]; networking.firewall = { @@ -111,6 +113,10 @@ in ''; }; }; + openiscsi = { + enable = true; + name = "${config.networking.hostName}-initiatorhost"; + }; }; users = {