From 792217f64011a90fc94552de8e83c8249d907998 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 28 Mar 2026 23:24:52 -0500 Subject: [PATCH] feat: add Nebula mesh network overlay Introduces a greg.nebula NixOS module and enables it across all managed hosts for the nebula.thehellings.com overlay (CIDR: 10.157.0.0/16). Architecture: - linode: lighthouse + relay (public internet, UDP 4242) - genesis: regular node + unsafe_routes router for 10.42.0.0/16 (home LAN) - hosea, isaiah, jeremiah, zeke, exodus: regular nodes with unsafe_routes pointing to genesis to reach the home LAN Changes: - modules/nixos/nebula.nix: new greg.nebula module - isLighthouse / isRelay options - unsafeRoutes option (tun.unsafe_routes) - routesSubnet option: enables IP forwarding + nftables masquerade NAT on the gateway host (genesis) so Nebula peers reach 10.42.0.0/16 - agenix secret reference per-host (secrets/nebula/.key.age) - opens UDP/4242 in the firewall - modules/nixos/default.nix: import nebula.nix - hosts/unstable/linode/default.nix: greg.nebula.isLighthouse = true - hosts/unstable/genesis/default.nix: greg.nebula.routesSubnet = "10.42.0.0/16" - hosts/unstable/{hosea,isaiah,jeremiah,zeke,exodus}/default.nix: greg.nebula.enable = true with unsafeRoutes via genesis - network.json: add nebulaIp field for each managed host - secrets/secrets.nix: declare nebula/.key.age entries - secrets/nebula/README.md: full PKI bootstrap guide (CA, certs, agenix) --- hosts/unstable/exodus/default.nix | 9 ++ hosts/unstable/genesis/default.nix | 6 + hosts/unstable/hosea/default.nix | 9 ++ hosts/unstable/isaiah/default.nix | 9 ++ hosts/unstable/jeremiah/default.nix | 9 ++ hosts/unstable/linode/default.nix | 4 + hosts/unstable/zeke/default.nix | 9 ++ modules/nixos/default.nix | 1 + modules/nixos/nebula.nix | 222 ++++++++++++++++++++++++++++ network.json | 56 +++++-- secrets/nebula/.gitkeep | 8 + secrets/nebula/README.md | 109 ++++++++++++++ secrets/secrets.nix | 10 ++ 13 files changed, 451 insertions(+), 10 deletions(-) create mode 100644 modules/nixos/nebula.nix create mode 100644 secrets/nebula/.gitkeep create mode 100644 secrets/nebula/README.md diff --git a/hosts/unstable/exodus/default.nix b/hosts/unstable/exodus/default.nix index 81da8e4..900e6d4 100644 --- a/hosts/unstable/exodus/default.nix +++ b/hosts/unstable/exodus/default.nix @@ -30,6 +30,15 @@ greg = { home = true; gnome.enable = true; + nebula = { + enable = true; + unsafeRoutes = [ + { + route = "10.42.0.0/16"; + via = "10.157.0.2"; # genesis + } + ]; + }; podman.enable = true; print.enable = true; tailscale = { diff --git a/hosts/unstable/genesis/default.nix b/hosts/unstable/genesis/default.nix index 4b45663..58d0e2a 100644 --- a/hosts/unstable/genesis/default.nix +++ b/hosts/unstable/genesis/default.nix @@ -35,6 +35,12 @@ in greg = { home = true; gnome.enable = false; + nebula = { + enable = true; + # genesis routes the home LAN (10.42.0.0/16) into the Nebula overlay. + # Sign genesis's cert with -subnets '10.42.0.0/16' (see secrets/nebula/README.md). + routesSubnet = "10.42.0.0/16"; + }; proxies = { }; }; diff --git a/hosts/unstable/hosea/default.nix b/hosts/unstable/hosea/default.nix index a9a06a5..9cf088f 100644 --- a/hosts/unstable/hosea/default.nix +++ b/hosts/unstable/hosea/default.nix @@ -49,6 +49,15 @@ in greg = { home = true; + nebula = { + enable = true; + unsafeRoutes = [ + { + route = "10.42.0.0/16"; + via = "10.157.0.2"; # genesis + } + ]; + }; proxies = { "jellyfin.home".target = "http://localhost:8096/"; "jellyfin.thehellings.lan".target = "http://localhost:8096/"; diff --git a/hosts/unstable/isaiah/default.nix b/hosts/unstable/isaiah/default.nix index 8e52f61..29ac835 100644 --- a/hosts/unstable/isaiah/default.nix +++ b/hosts/unstable/isaiah/default.nix @@ -42,6 +42,15 @@ vip = metadata.hosts.${config.networking.hostName}.ip; priority = 255; }; + nebula = { + enable = true; + unsafeRoutes = [ + { + route = "10.42.0.0/16"; + via = "10.157.0.2"; # genesis + } + ]; + }; podman.enable = true; tailscale = { enable = true; diff --git a/hosts/unstable/jeremiah/default.nix b/hosts/unstable/jeremiah/default.nix index 90a3cd1..a2380a3 100644 --- a/hosts/unstable/jeremiah/default.nix +++ b/hosts/unstable/jeremiah/default.nix @@ -85,6 +85,15 @@ in vip = ip; priority = 254; }; + nebula = { + enable = true; + unsafeRoutes = [ + { + route = "10.42.0.0/16"; + via = "10.157.0.2"; # genesis + } + ]; + }; tailscale = { enable = true; tags = [ "home" ]; diff --git a/hosts/unstable/linode/default.nix b/hosts/unstable/linode/default.nix index 3c9cb84..b25127b 100644 --- a/hosts/unstable/linode/default.nix +++ b/hosts/unstable/linode/default.nix @@ -26,6 +26,10 @@ greg = { home = false; linode.enable = true; + nebula = { + enable = true; + isLighthouse = true; + }; proxies."immich.thehellings.com" = { genAliases = false; target = "http://localhost:${builtins.toString config.services.immich-public-proxy.port}"; diff --git a/hosts/unstable/zeke/default.nix b/hosts/unstable/zeke/default.nix index 37fae9e..5b94e3c 100644 --- a/hosts/unstable/zeke/default.nix +++ b/hosts/unstable/zeke/default.nix @@ -26,6 +26,15 @@ vipInterface = "enp12s0"; priority = 253; }; + nebula = { + enable = true; + unsafeRoutes = [ + { + route = "10.42.0.0/16"; + via = "10.157.0.2"; # genesis + } + ]; + }; remote-builder.enable = true; runner = { enable = true; diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index ed4359a..9296ee6 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -22,6 +22,7 @@ ./router.nix ./rpi4.nix ./syncthing.nix + ./nebula.nix ./tailscale.nix ./vmdev.nix ]; diff --git a/modules/nixos/nebula.nix b/modules/nixos/nebula.nix new file mode 100644 index 0000000..45fcd67 --- /dev/null +++ b/modules/nixos/nebula.nix @@ -0,0 +1,222 @@ +{ + config, + lib, + metadata, + ... +}: + +# Nebula overlay mesh network module (greg namespace) +# +# This module configures a Nebula node for the nebula.thehellings.com overlay. +# CIDR: 10.157.0.0/16 +# Lighthouse: linode (public internet, acts as relay too) +# +# Each host requires: +# secrets/nebula/.key.age - encrypted private key +# secrets/nebula/.crt - certificate (public, unencrypted in repo) +# secrets/nebula/ca.crt - CA certificate (public, unencrypted in repo) +# +# The nebula IP for each host must be set in network.json under +# hosts..nebulaIp (e.g. "10.157.0.1") +# +# Lighthouse address is the public IP/DNS of linode. Update the +# `lighthouseAddrs` option below or override per-host if it changes. + +let + cfg = config.greg.nebula; + nebulaDomain = "nebula.thehellings.com"; + # Linode's public address — used by all non-lighthouse hosts to reach it. + # Override with greg.nebula.lighthouseAddr if the public IP ever changes. + defaultLighthouseAddr = "linode.${nebulaDomain}"; +in +{ + options.greg.nebula = { + enable = lib.mkEnableOption "Nebula overlay mesh network"; + + isLighthouse = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Whether this host is a Nebula lighthouse/relay node"; + }; + + isRelay = lib.mkOption { + type = lib.types.bool; + default = cfg.isLighthouse; + description = "Whether this host acts as a relay (am_relay). Defaults to true when isLighthouse is true."; + }; + + nebulaIp = lib.mkOption { + type = lib.types.str; + description = "This host's Nebula overlay IP (e.g. 10.157.0.1)"; + default = + let + hostData = metadata.hosts.${config.networking.hostName} or { }; + in + hostData.nebulaIp or (throw "greg.nebula.nebulaIp must be set for host ${config.networking.hostName}"); + }; + + lighthouseAddr = lib.mkOption { + type = lib.types.str; + default = defaultLighthouseAddr; + description = "Public address (host:port) used to reach the lighthouse from non-lighthouse hosts"; + }; + + lighthouseNebulaIp = lib.mkOption { + type = lib.types.str; + default = + let + linodeData = metadata.hosts.linode or { }; + in + linodeData.nebulaIp or "10.157.0.1"; + description = "Nebula overlay IP of the lighthouse host"; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 4242; + description = "UDP port Nebula listens on"; + }; + + # unsafe_routes: allow non-Nebula subnets to be routed through this host. + # Used on genesis to expose 10.42.0.0/16 (the home LAN) to the overlay. + unsafeRoutes = lib.mkOption { + type = lib.types.listOf ( + lib.types.submodule { + options = { + route = lib.mkOption { + type = lib.types.str; + description = "CIDR to route through this host (e.g. 10.42.0.0/16)"; + }; + via = lib.mkOption { + type = lib.types.str; + description = "Nebula overlay IP of the host that provides the route"; + }; + }; + } + ); + default = [ ]; + description = "List of unsafe_routes to configure on this host (for reaching non-Nebula subnets)"; + }; + + # Whether this host IS the router for an unsafe subnet + # (enables IP forwarding + nftables masquerade for the home LAN) + routesSubnet = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + When set, this host will route traffic from the Nebula overlay + to this subnet. Enables IP forwarding and nftables masquerade. + Example: "10.42.0.0/16" + ''; + }; + }; + + config = lib.mkIf cfg.enable { + # agenix: decrypt this host's Nebula private key at boot + age.secrets."nebula-${config.networking.hostName}-key" = { + file = ../../secrets/nebula/${config.networking.hostName}.key.age; + # nebula service runs as root, key owned by root is fine + mode = "0400"; + }; + + services.nebula.networks.${nebulaDomain} = { + enable = true; + + # CA certificate (public — lives unencrypted in the repo) + ca = ../../secrets/nebula/ca.crt; + + # Host certificate (public — lives unencrypted in the repo) + cert = ../../secrets/nebula/${config.networking.hostName}.crt; + + # Private key (agenix-decrypted at runtime) + key = config.age.secrets."nebula-${config.networking.hostName}-key".path; + + # Static host map: tell every node where the lighthouse lives + staticHostMap = { + "${cfg.lighthouseNebulaIp}" = [ "${cfg.lighthouseAddr}:${toString cfg.port}" ]; + }; + + isLighthouse = cfg.isLighthouse; + isRelay = cfg.isRelay; + + listen = { + host = "0.0.0.0"; + port = cfg.port; + }; + + lighthouses = lib.optionals (!cfg.isLighthouse) [ cfg.lighthouseNebulaIp ]; + + relays = lib.optionals (!cfg.isLighthouse && !cfg.isRelay) [ cfg.lighthouseNebulaIp ]; + + tun = { + # Interface name: nebula. + dev = "nebula0"; + # unsafe_routes for hosts that need access to a non-Nebula subnet + unsafeRoutes = cfg.unsafeRoutes; + }; + + # Firewall: permissive defaults — tighten per-host as desired + firewall = { + outbound = [ + { + port = "any"; + proto = "any"; + host = "any"; + } + ]; + inbound = + [ + # Allow ICMP (ping) from any Nebula peer + { + port = "any"; + proto = "icmp"; + host = "any"; + } + # Allow all traffic from within the Nebula overlay + { + port = "any"; + proto = "any"; + host = "any"; + } + ] + # When routing an unsafe subnet, allow inbound traffic destined + # for that subnet from any Nebula peer (local_cidr scopes it) + ++ lib.optionals (cfg.routesSubnet != null) [ + { + port = "any"; + proto = "any"; + host = "any"; + local_cidr = cfg.routesSubnet; + } + ]; + }; + }; + + # Open the Nebula UDP port in the firewall + networking.firewall.allowedUDPPorts = [ cfg.port ]; + + # When this host routes traffic to a non-Nebula subnet, enable IP + # forwarding and add nftables masquerade rules (see unsafe_routes guide). + boot.kernel.sysctl = lib.mkIf (cfg.routesSubnet != null) { + "net.ipv4.ip_forward" = lib.mkDefault "1"; + }; + + networking.nftables.tables = lib.mkIf (cfg.routesSubnet != null) { + nebula_routing = { + family = "ip"; + content = '' + chain postrouting { + type nat hook postrouting priority srcnat; policy accept; + ip saddr 10.157.0.0/16 ip daddr ${cfg.routesSubnet} counter masquerade + } + + chain forward { + type filter hook forward priority filter; policy accept; + ct state related,established counter accept + iifname "nebula0" ip saddr 10.157.0.0/16 ip daddr ${cfg.routesSubnet} counter accept + } + ''; + }; + }; + }; +} diff --git a/network.json b/network.json index d61870c..fd75de4 100644 --- a/network.json +++ b/network.json @@ -16,21 +16,29 @@ "ip": null, "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3", "ts": "100.70.99.91", - "system": "x86_64-linux" + "system": "x86_64-linux", + "nebulaIp": "10.157.0.7" }, "genesis": { "ip": "10.42.1.5", "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO/CGE2rqlL2Qr0TJkwQMsHFSDkhGWlrUlvfcvcylO1n", "ts": "100.96.198.104", "system": "x86_64-linux", - "tags": ["router", "server"] + "tags": [ + "router", + "server" + ], + "nebulaIp": "10.157.0.2" }, "hosea": { "ip": "10.42.1.7", "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKLIwkTTXA56sUlUjEulXXZRvZy5H4a5ZwgKWLlpkQDz", "ts": "100.68.203.1", "system": "x86_64-linux", - "tags": ["server"] + "tags": [ + "server" + ], + "nebulaIp": "10.157.0.3" }, "icdm-root": { "external": true, @@ -42,8 +50,16 @@ "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHleYKtfV4W1Z63Ysu9w5Rbglqlz4F92YcZoMkucoTNf", "ts": "100.84.183.79", "system": "x86_64-linux", - "systems": ["x86_64-linux", "aarch64-linux"], - "tags": ["builder", "kube", "server"] + "systems": [ + "x86_64-linux", + "aarch64-linux" + ], + "tags": [ + "builder", + "kube", + "server" + ], + "nebulaIp": "10.157.0.4" }, "iso": { "external": true, @@ -59,8 +75,16 @@ "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOjQjXq9WYU2Ki27BR9WwJ4ZruS/lJXbjC1b0Q42Adi0", "ts": "100.102.186.39", "system": "x86_64-linux", - "systems": ["x86_64-linux", "aarch64-linux"], - "tags": ["builder", "kube", "server"] + "systems": [ + "x86_64-linux", + "aarch64-linux" + ], + "tags": [ + "builder", + "kube", + "server" + ], + "nebulaIp": "10.157.0.5" }, "joel": { "external": true, @@ -72,7 +96,11 @@ "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMv9Zud3kZOl86gtmkn+uj3D4kiXWDPtyUL02VVLNR4Q", "ts": "100.109.86.8", "system": "x86_64-linux", - "tags": ["public", "server"] + "tags": [ + "public", + "server" + ], + "nebulaIp": "10.157.0.1" }, "MacBook-Pro.local": { "external": true, @@ -108,8 +136,16 @@ "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOos0zQePsa+T6Z2dsKbPOvEdrBQ8a6mx3s7pN6ysCI0", "ts": "100.90.74.19", "system": "x86_64-linux", - "systems": ["x86_64-linux", "aarch64-linux"], - "tags": ["builder", "kube", "server"] + "systems": [ + "x86_64-linux", + "aarch64-linux" + ], + "tags": [ + "builder", + "kube", + "server" + ], + "nebulaIp": "10.157.0.6" } } } diff --git a/secrets/nebula/.gitkeep b/secrets/nebula/.gitkeep new file mode 100644 index 0000000..67ef0e0 --- /dev/null +++ b/secrets/nebula/.gitkeep @@ -0,0 +1,8 @@ +# Files that belong here (committed to repo): +# ca.crt — Nebula CA certificate (public) +# *.crt — per-host certificates (public) +# *.key.age — agenix-encrypted private keys (safe to commit) +# +# Files that must NEVER be committed: +# ca.key — CA private key (keep offline!) +# *.key — plaintext host private keys (encrypt with agenix first) diff --git a/secrets/nebula/README.md b/secrets/nebula/README.md new file mode 100644 index 0000000..abdf6a4 --- /dev/null +++ b/secrets/nebula/README.md @@ -0,0 +1,109 @@ +# Nebula PKI Bootstrap Guide + +This directory holds the Nebula CA certificate, host certificates, and +agenix-encrypted private keys for the `nebula.thehellings.com` overlay network. + +## CIDR +`10.157.0.0/16` + +## Host IP Assignments + +| Host | Nebula IP | Role | +|-----------|-------------|----------------------| +| linode | 10.157.0.1 | Lighthouse + relay | +| genesis | 10.157.0.2 | LAN router (unsafe) | +| hosea | 10.157.0.3 | Regular node | +| isaiah | 10.157.0.4 | Regular node | +| jeremiah | 10.157.0.5 | Regular node | +| zeke | 10.157.0.6 | Regular node | +| exodus | 10.157.0.7 | Regular node (laptop) | + +--- + +## Step 1 — Install nebula-cert + +```bash +nix shell nixpkgs#nebula +``` + +## Step 2 — Create the CA + +Run once; keep `ca.key` offline/safe (do NOT commit it): + +```bash +nebula-cert ca -name "thehellings.com" -out-crt ca.crt -out-key ca.key +``` + +Commit `ca.crt` (public) to the repo at `secrets/nebula/ca.crt`. +Store `ca.key` securely (password manager / offline). + +## Step 3 — Sign host certificates + +For most hosts (no subnet routing): +```bash +nebula-cert sign -ca-crt ca.crt -ca-key ca.key \ + -name \ + -ip /16 \ + -out-crt secrets/nebula/.crt \ + -out-key secrets/nebula/.key +``` + +For **genesis** (routes the home LAN `10.42.0.0/16`), add `-subnets`: +```bash +nebula-cert sign -ca-crt ca.crt -ca-key ca.key \ + -name genesis \ + -ip 10.157.0.2/16 \ + -subnets '10.42.0.0/16' \ + -out-crt secrets/nebula/genesis.crt \ + -out-key secrets/nebula/genesis.key +``` + +Commit `*.crt` files (public) to the repo. +Do NOT commit raw `*.key` files — encrypt them first (Step 4). + +## Step 4 — Encrypt private keys with agenix + +From the repo root: +```bash +cd nixos +for host in linode genesis hosea isaiah jeremiah zeke exodus; do + agenix -e secrets/nebula/${host}.key.age < secrets/nebula/${host}.key + rm secrets/nebula/${host}.key # remove plaintext key +done +``` + +The `secrets/secrets.nix` file already declares the `.key.age` recipients. + +## Step 5 — Configure linode's public DNS / firewall + +- Add a DNS A record: `linode.nebula.thehellings.com` → linode's public IP +- Open UDP port `4242` in linode's firewall / Linode Cloud Firewall rules + +## Step 6 — Deploy + +```bash +colmena apply --on linode # lighthouse first +colmena apply # rest of the fleet +``` + +## Verifying + +```bash +# From any host, ping another by Nebula IP +ping 10.157.0.2 # genesis + +# From any host, reach home LAN via unsafe_routes +ping 10.42.1.1 # UDM Pro (via genesis) +``` + +## Notes + +- `ca.crt` is public and lives in the repo unencrypted. +- `*.crt` (host certs) are public and live in the repo unencrypted. +- `*.key.age` are agenix-encrypted private keys (in `secrets/nebula/`). +- The NixOS module (`modules/nixos/nebula.nix`) references these paths directly. +- The lighthouse (`linode`) has `isLighthouse = true` and `isRelay = true`. +- `genesis` has `routesSubnet = "10.42.0.0/16"` which enables IP forwarding + and nftables masquerade so Nebula peers can reach the home LAN. +- All other hosts have `unsafeRoutes` pointing to genesis (10.157.0.2) for + the 10.42.0.0/16 subnet. diff --git a/secrets/secrets.nix b/secrets/secrets.nix index cffce51..bba6700 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -137,4 +137,14 @@ in "compose/attic.env.age".publicKeys = everyone; "grafana-api-token.age".publicKeys = everyone; + + # Nebula mesh network — one private key per host, encrypted to that host's + # system key + all user keys so Greg can (re)encrypt them from any machine. + "nebula/genesis.key.age".publicKeys = everyone; + "nebula/hosea.key.age".publicKeys = everyone; + "nebula/isaiah.key.age".publicKeys = everyone; + "nebula/jeremiah.key.age".publicKeys = everyone; + "nebula/linode.key.age".publicKeys = everyone; + "nebula/zeke.key.age".publicKeys = everyone; + "nebula/exodus.key.age".publicKeys = everyone; }