Add keepalived to cluster

Add keepalived for the Kubernetes cluster
Set VIP to 10.42.5.1
Set *.cluster DNS resolution to this IP address
But do not move existing *.kubernetes targets, yet, as there is a need
to ensure that subnet routers are properly configured

hosts/genesis/net/hosts

@@ -23,6 +23,9 @@
 10.42.4.2 jellyfin jellyfin.thehellings.lan vm-jellyfin vm-jellyfin.thehellings.lan
 10.42.4.3 git gitlab git.thehellings.lan gitlab.thehellings.lan
 
+# VIP
+10.42.5.1 pgadmin.cluter postgres.cluster matrix.cluster
+
 # IPMI
 10.42.100.6 isaiahbmc isaiahbmc.thehellings.lan
 

hosts/isaiah/default.nix

@@ -35,7 +35,11 @@
   };
 
   greg = {
-    kubernetes.enable = true;
+    kubernetes = {
+      enable = true;
+      vipInterface = "enp38s0";
+      priority = 255;
+    };
     tailscale.enable = true;
     remote-builder.enable = true;
   };

hosts/jeremiah/default.nix

@@ -75,7 +75,11 @@ in
 
   greg = {
     home = true;
-    kubernetes.enable = true;
+    kubernetes = {
+      enable = true;
+      vipInterface = "br0";
+      priority = 254;
+    };
     tailscale.enable = true;
     remote-builder.enable = true;
   };

hosts/zeke/default.nix

@@ -17,7 +17,11 @@
 
   greg = {
     tailscale.enable = true;
-    kubernetes.enable = true;
+    kubernetes = {
+      enable = true;
+      vipInterface = "enp12s0";
+      priority = 253;
+    };
     remote-builder.enable = true;
   };
 

modules/nixos/kubernetes.nix

@@ -16,6 +16,16 @@ in
         default = false;
         description = "Whether to run the Kubernetes agent only";
       };
+      priority = lib.mkOption {
+        type = lib.types.int;
+        default = 1;
+        description = "VIP priority";
+      };
+      vipInterface = lib.mkOption {
+        type = lib.types.str;
+        default = null;
+        description = "Interface to attach VIP for clustering onto";
+      };
     };
   };
 
@@ -55,7 +65,8 @@ in
       allowedUDPPorts = [ 8472 ];
     };
 
-    services.k3s = {
+    services = {
+      k3s = {
         enable = true;
         role = if cfg.agentOnly then "agent" else "server";
         tokenFile = config.age.secrets.kubernetesToken.path;
@@ -71,6 +82,36 @@ in
         ];
         serverAddr = lib.mkIf (config.networking.hostName != "isaiah") "https://isaiah.home:6443";
       };
+      keepalived =
+        let
+          ip = (builtins.head config.networking.interfaces."${cfg.vipInterface}".ipv4.addresses).address;
+        in
+        {
+          enable = true;
+          openFirewall = true;
+          vrrpInstances.kubernetes = {
+            interface = cfg.vipInterface;
+            priority = cfg.priority;
+            state = if (config.networking.hostName == "isaiah") then "MASTER" else "BACKUP";
+            virtualIps = [
+              {
+                addr = "10.42.5.1/16";
+                dev = cfg.vipInterface;
+              }
+            ];
+            virtualRouterId = 77;
+            unicastPeers = lib.filter (v: v != ip) [
+              "10.42.1.6"
+              "10.42.1.8"
+              "10.42.1.13"
+            ];
+            unicastSrcIp = ip;
+            extraConfig = ''
+              advert_int 1
+            '';
+          };
+        };
+    };
 
     users = {
       groups.kubeconfig = { };