From 9940e89d2ecdbb11a006e95bc075c77387ad05fc Mon Sep 17 00:00:00 2001
From: Greg Hellings <greg.hellings@gmail.com>
Date: Wed, 2 Jul 2025 23:07:08 -0500
Subject: [PATCH] Add keepalived to cluster

Add keepalived for the Kubernetes cluster
Set VIP to 10.42.5.1
Set *.cluster DNS resolution to this IP address
But do not move existing *.kubernetes targets, yet, as there is a need
to ensure that subnet routers are properly configured
---
 hosts/genesis/net/hosts      |  3 ++
 hosts/isaiah/default.nix     |  6 ++-
 hosts/jeremiah/default.nix   |  6 ++-
 hosts/zeke/default.nix       |  6 ++-
 modules/nixos/kubernetes.nix | 71 ++++++++++++++++++++++++++++--------
 5 files changed, 74 insertions(+), 18 deletions(-)

diff --git a/hosts/genesis/net/hosts b/hosts/genesis/net/hosts
index 53dafd0..5228fd0 100644
--- a/hosts/genesis/net/hosts
+++ b/hosts/genesis/net/hosts
@@ -23,6 +23,9 @@
 10.42.4.2 jellyfin jellyfin.thehellings.lan vm-jellyfin vm-jellyfin.thehellings.lan
 10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan
 
+# VIP
+10.42.5.1 pgadmin.cluter postgres.cluster matrix.cluster
+
 # IPMI
 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan
 
diff --git a/hosts/isaiah/default.nix b/hosts/isaiah/default.nix
index 31cc0dc..9b9825d 100644
--- a/hosts/isaiah/default.nix
+++ b/hosts/isaiah/default.nix
@@ -35,7 +35,11 @@
   };
 
   greg = {
-    kubernetes.enable = true;
+    kubernetes = {
+      enable = true;
+      vipInterface = "enp38s0";
+      priority = 255;
+    };
     tailscale.enable = true;
     remote-builder.enable = true;
   };
diff --git a/hosts/jeremiah/default.nix b/hosts/jeremiah/default.nix
index 5b26fd2..d4dcfeb 100644
--- a/hosts/jeremiah/default.nix
+++ b/hosts/jeremiah/default.nix
@@ -75,7 +75,11 @@ in
 
   greg = {
     home = true;
-    kubernetes.enable = true;
+    kubernetes = {
+      enable = true;
+      vipInterface = "br0";
+      priority = 254;
+    };
     tailscale.enable = true;
     remote-builder.enable = true;
   };
diff --git a/hosts/zeke/default.nix b/hosts/zeke/default.nix
index c1df80a..e0cddf6 100644
--- a/hosts/zeke/default.nix
+++ b/hosts/zeke/default.nix
@@ -17,7 +17,11 @@
 
   greg = {
     tailscale.enable = true;
-    kubernetes.enable = true;
+    kubernetes = {
+      enable = true;
+      vipInterface = "enp12s0";
+      priority = 253;
+    };
     remote-builder.enable = true;
   };
 
diff --git a/modules/nixos/kubernetes.nix b/modules/nixos/kubernetes.nix
index d3aed94..cd0b647 100644
--- a/modules/nixos/kubernetes.nix
+++ b/modules/nixos/kubernetes.nix
@@ -16,6 +16,16 @@ in
         default = false;
         description = "Whether to run the Kubernetes agent only";
       };
+      priority = lib.mkOption {
+        type = lib.types.int;
+        default = 1;
+        description = "VIP priority";
+      };
+      vipInterface = lib.mkOption {
+        type = lib.types.str;
+        default = null;
+        description = "Interface to attach VIP for clustering onto";
+      };
     };
   };
 
@@ -55,21 +65,52 @@ in
       allowedUDPPorts = [ 8472 ];
     };
 
-    services.k3s = {
-      enable = true;
-      role = if cfg.agentOnly then "agent" else "server";
-      tokenFile = config.age.secrets.kubernetesToken.path;
-      extraFlags = [
-        "--cluster-cidr=10.211.0.0/16"
-        "--service-cidr=10.221.0.0/16"
-        "--write-kubeconfig-mode 0640"
-        "--write-kubeconfig-group kubeconfig"
-        "--resolv-conf=/etc/resolv.conf"
-        "--tls-san ${config.networking.hostName}.home"
-        "--tls-san ${config.networking.hostName}.thehellings.lan"
-        "--tls-san ${config.networking.hostName}.shire-zebra.ts.net"
-      ];
-      serverAddr = lib.mkIf (config.networking.hostName != "isaiah") "https://isaiah.home:6443";
+    services = {
+      k3s = {
+        enable = true;
+        role = if cfg.agentOnly then "agent" else "server";
+        tokenFile = config.age.secrets.kubernetesToken.path;
+        extraFlags = [
+          "--cluster-cidr=10.211.0.0/16"
+          "--service-cidr=10.221.0.0/16"
+          "--write-kubeconfig-mode 0640"
+          "--write-kubeconfig-group kubeconfig"
+          "--resolv-conf=/etc/resolv.conf"
+          "--tls-san ${config.networking.hostName}.home"
+          "--tls-san ${config.networking.hostName}.thehellings.lan"
+          "--tls-san ${config.networking.hostName}.shire-zebra.ts.net"
+        ];
+        serverAddr = lib.mkIf (config.networking.hostName != "isaiah") "https://isaiah.home:6443";
+      };
+      keepalived =
+        let
+          ip = (builtins.head config.networking.interfaces."${cfg.vipInterface}".ipv4.addresses).address;
+        in
+        {
+          enable = true;
+          openFirewall = true;
+          vrrpInstances.kubernetes = {
+            interface = cfg.vipInterface;
+            priority = cfg.priority;
+            state = if (config.networking.hostName == "isaiah") then "MASTER" else "BACKUP";
+            virtualIps = [
+              {
+                addr = "10.42.5.1/16";
+                dev = cfg.vipInterface;
+              }
+            ];
+            virtualRouterId = 77;
+            unicastPeers = lib.filter (v: v != ip) [
+              "10.42.1.6"
+              "10.42.1.8"
+              "10.42.1.13"
+            ];
+            unicastSrcIp = ip;
+            extraConfig = ''
+              advert_int 1
+            '';
+          };
+        };
     };
 
     users = {