141 lines
3.6 KiB
Nix
141 lines
3.6 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.greg.kubernetes;
|
|
cert-manager = pkgs.fetchurl {
|
|
url = "https://github.com/cert-manager/cert-manager/releases/download/v1.20.2/cert-manager.yaml";
|
|
sha256 = "sha256-HOEcrpEq3sxp5rtiNDX6/J7SFQX57/+YvXHXuA8B2x8=";
|
|
};
|
|
flux = pkgs.fetchurl {
|
|
url = "https://github.com/fluxcd/flux2/releases/download/v2.7.2/install.yaml";
|
|
sha256 = "sha256-Qs1qJmgZm8q9xZsORjT/N/wzpbWVVODXtzDpjnAYMuQ=";
|
|
};
|
|
in
|
|
{
|
|
options.greg = {
|
|
kubernetes = {
|
|
enable = lib.mkEnableOption "kubernetes";
|
|
agentOnly = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = "Whether to run the Kubernetes agent only";
|
|
};
|
|
priority = lib.mkOption {
|
|
type = lib.types.int;
|
|
default = 1;
|
|
description = "VIP priority";
|
|
};
|
|
vip = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = null;
|
|
description = "The IP address of this host";
|
|
};
|
|
vipInterface = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = null;
|
|
description = "Interface to attach VIP for clustering onto";
|
|
};
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
age.secrets = {
|
|
bw_secret.file = ../../secrets/kubernetes/bw_secret.age;
|
|
dendrite_key.file = ../../secrets/dendrite_key.age;
|
|
kubernetesToken.file = ../../secrets/kubernetes/kubernetesToken.age;
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.etcd
|
|
pkgs.fluxcd
|
|
pkgs.kubectl-cnpg
|
|
pkgs.kubernetes-helm
|
|
pkgs.kustomize
|
|
pkgs.k9s
|
|
pkgs.openiscsi
|
|
pkgs.nfs-utils # Needed for Longhorn
|
|
];
|
|
|
|
networking.firewall = {
|
|
allowedTCPPorts = [
|
|
80
|
|
443
|
|
5432
|
|
6443
|
|
8080
|
|
8443
|
|
10250
|
|
]
|
|
++ (
|
|
if cfg.agentOnly then
|
|
[ ]
|
|
else
|
|
[
|
|
2379
|
|
2380
|
|
]
|
|
);
|
|
allowedUDPPorts = [ 8472 ];
|
|
};
|
|
|
|
services = {
|
|
k3s = {
|
|
enable = true;
|
|
extraFlags = [
|
|
"--cluster-cidr=10.211.0.0/16"
|
|
"--service-cidr=10.221.0.0/16"
|
|
"--write-kubeconfig-mode 0640"
|
|
"--write-kubeconfig-group kubeconfig"
|
|
"--resolv-conf=/etc/resolv.conf"
|
|
"--node-label node.longhorn.io/create-default-disk=config"
|
|
"--supervisor-metrics=true"
|
|
"--tls-san ${config.networking.hostName}.thehellings.lan"
|
|
"--tls-san ${config.networking.hostName}.shire-zebra.ts.net"
|
|
];
|
|
manifests = {
|
|
cert-manager.source = cert-manager;
|
|
flux.source = flux;
|
|
node-annotations.source = ../../manifests/auto/nodes.yaml;
|
|
};
|
|
role = if cfg.agentOnly then "agent" else "server";
|
|
serverAddr = lib.mkIf (
|
|
config.networking.hostName != "isaiah"
|
|
) "https://isaiah.shire-zebra.ts.net:6443";
|
|
tokenFile = config.age.secrets.kubernetesToken.path;
|
|
};
|
|
keepalived = {
|
|
enable = true;
|
|
openFirewall = true;
|
|
vrrpInstances.kubernetes = {
|
|
interface = cfg.vipInterface;
|
|
priority = cfg.priority;
|
|
state = if (config.networking.hostName == "isaiah") then "MASTER" else "BACKUP";
|
|
virtualIps = [
|
|
{
|
|
addr = "10.42.5.1/16";
|
|
dev = cfg.vipInterface;
|
|
}
|
|
];
|
|
virtualRouterId = 77;
|
|
extraConfig = ''
|
|
advert_int 1
|
|
'';
|
|
};
|
|
};
|
|
openiscsi = {
|
|
enable = true;
|
|
name = "${config.networking.hostName}-initiatorhost";
|
|
};
|
|
};
|
|
|
|
users = {
|
|
groups.kubeconfig = { };
|
|
users.greg.extraGroups = [ "kubeconfig" ];
|
|
};
|
|
};
|
|
}
|