From 28977235a1d5753397bf583c56f364d7a0221884 Mon Sep 17 00:00:00 2001 From: "Emily (Agent)" Date: Sat, 8 Aug 2026 02:49:50 -0500 Subject: [PATCH] chore: clean up builder2, Ceph module, normalize Darwin host symlinks - Remove builder2 (retired host): dangling network.json entry and empty home/hosts/builder2 stub. Its old IP (10.42.1.17) is already correctly owned by pve4. - Remove the abandoned Ceph module (modules/nixos/ceph.nix) and its unencrypted plaintext keyring files under secrets/. No host ever enabled services.ceph-benaco; the keyrings were dead, unencrypted credentials sitting in the repo. - Normalize Darwin host identity: IVR and Lithic are the only two physical Darwin machines. All other darwin/hosts/* names are now symlinks to whichever of the two they represent, matching the DHCP name variations nix-darwin sees depending on network: gregory -> ivr gregory.hellings-mbp -> ivr MacBook-Prolocal -> ivr gregs-MacBook-Pro-16-inch-Nov-2024 -> lithic li -> lithic (pre-existing) This lets each machine's config be maintained once regardless of what hostname it currently advertises. --- darwin/hosts/MacBook-Prolocal | 2 +- darwin/hosts/gregory | 1 + darwin/hosts/gregory.hellings-mbp | 2 +- .../hosts/gregs-MacBook-Pro-16-inch-Nov-2024 | 1 + darwin/hosts/ivr | 1 - darwin/hosts/{gregory => ivr}/default.nix | 0 home/hosts/builder2/default.nix | 1 - modules/nixos/ceph.nix | 927 ------------------ modules/nixos/default.nix | 1 - network.json | 4 - secrets/home.client.admin.keyring | 6 - secrets/home.keyring | 0 secrets/home.mon.keyring | 13 - secrets/home.osd-bootstrap.keyring | 4 - 14 files changed, 4 insertions(+), 959 deletions(-) create mode 120000 darwin/hosts/gregory create mode 120000 darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024 delete mode 120000 darwin/hosts/ivr rename darwin/hosts/{gregory => ivr}/default.nix (100%) delete mode 100644 home/hosts/builder2/default.nix delete mode 100644 modules/nixos/ceph.nix delete mode 100644 secrets/home.client.admin.keyring delete mode 100644 secrets/home.keyring delete mode 100644 secrets/home.mon.keyring delete mode 100644 secrets/home.osd-bootstrap.keyring diff --git a/darwin/hosts/MacBook-Prolocal b/darwin/hosts/MacBook-Prolocal index c12c63e..f6cb244 120000 --- a/darwin/hosts/MacBook-Prolocal +++ b/darwin/hosts/MacBook-Prolocal @@ -1 +1 @@ -gregory \ No newline at end of file +ivr \ No newline at end of file diff --git a/darwin/hosts/gregory b/darwin/hosts/gregory new file mode 120000 index 0000000..f6cb244 --- /dev/null +++ b/darwin/hosts/gregory @@ -0,0 +1 @@ +ivr \ No newline at end of file diff --git a/darwin/hosts/gregory.hellings-mbp b/darwin/hosts/gregory.hellings-mbp index c12c63e..f6cb244 120000 --- a/darwin/hosts/gregory.hellings-mbp +++ b/darwin/hosts/gregory.hellings-mbp @@ -1 +1 @@ -gregory \ No newline at end of file +ivr \ No newline at end of file diff --git a/darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024 b/darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024 new file mode 120000 index 0000000..45a5001 --- /dev/null +++ b/darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024 @@ -0,0 +1 @@ +lithic \ No newline at end of file diff --git a/darwin/hosts/ivr b/darwin/hosts/ivr deleted file mode 120000 index ed27f59..0000000 --- a/darwin/hosts/ivr +++ /dev/null @@ -1 +0,0 @@ -gregory/ \ No newline at end of file diff --git a/darwin/hosts/gregory/default.nix b/darwin/hosts/ivr/default.nix similarity index 100% rename from darwin/hosts/gregory/default.nix rename to darwin/hosts/ivr/default.nix diff --git a/home/hosts/builder2/default.nix b/home/hosts/builder2/default.nix deleted file mode 100644 index 6462967..0000000 --- a/home/hosts/builder2/default.nix +++ /dev/null @@ -1 +0,0 @@ -{...}: {} diff --git a/modules/nixos/ceph.nix b/modules/nixos/ceph.nix deleted file mode 100644 index 7bc7676..0000000 --- a/modules/nixos/ceph.nix +++ /dev/null @@ -1,927 +0,0 @@ -# This is a good source for a Ceph dealio -# https://gist.github.com0/nh2/13425a1f18b4c1ce82edb63c10b163c9 -{ - config, - lib, - pkgs, - ... -}: - -with lib; - -let - cfg = config.services.ceph-benaco; - commaSep = builtins.concatStringsSep ","; - - ensureUnitExists = - c': name: - let - in - #unitName = (builtins.elemAt (builtins.split "\\." name) 0); - if c'.systemd.services ? unitName then name else name; # "Unable to locate ${name} at ${commaSep (builtins.attrNames c')}"; -in - -{ - - ###### interface - - options = { - - services.ceph-benaco = { - - enable = mkEnableOption "Ceph distributed filesystem"; - - package = mkOption { - type = types.package; - default = pkgs.ceph; - defaultText = literalExpression "pkgs.ceph-benaco"; - description = "Ceph package to use."; - }; - - fsid = mkOption { - type = types.str; - description = "Unique cluster identifier."; - }; - - clusterName = mkOption { - type = types.str; - description = "Cluster name."; - default = "ceph"; - }; - - initialMonitors = mkOption { - type = types.listOf ( - types.submodule { - options = { - hostname = mkOption { - type = types.str; - description = "Initial monitor hostname."; - }; - - ipAddress = mkOption { - type = types.str; - description = "Initial monitor IP address."; - }; - }; - } - ); - description = "Initial monitors."; - }; - - mdsNodes = mkOption { - type = types.listOf ( - types.submodule { - options = { - hostname = mkOption { - type = types.str; - description = "MDS hostname."; - }; - - ipAddress = mkOption { - type = types.str; - description = "MDS IP address."; - }; - }; - } - ); - description = "MDS nodes."; - }; - - publicNetworks = mkOption { - type = types.listOf types.str; - description = "Public network(s) of the cluster."; - }; - - clusterNetworks = mkOption { - type = types.listOf types.str; - description = "Cluster backend networks for OSD sync"; - }; - - adminKeyring = mkOption { - type = types.path; - description = "Ceph admin keyring to install on the machine."; - }; - - monitor = { - enable = mkEnableOption "Activate a Ceph monitor on this machine."; - - initialKeyring = mkOption { - type = types.path; - description = "Keyring file to use when initializing a new monitor"; - example = "/path/to/ceph.mon.keyring"; - }; - - nodeName = mkOption { - type = types.str; - description = "Ceph monitor node name."; - example = "node1"; - }; - - bindAddr = mkOption { - type = types.str; - description = "IP address that the OSDs shall bind to."; - example = "10.0.0.1"; - }; - - advertisedPublicAddr = mkOption { - type = types.str; - description = "IP address that the monitor shall advertise."; - example = "10.0.0.1"; - }; - }; - - manager = { - enable = mkEnableOption "Activate a Ceph manager on this machine."; - - nodeName = mkOption { - type = types.str; - description = "Ceph manager node name."; - example = "node1"; - }; - }; - - osdBindAddr = mkOption { - type = types.str; - description = "IP address that the OSDs shall bind to."; - example = "10.0.0.1"; - }; - - osdAdvertisedPublicAddr = mkOption { - type = types.str; - description = "IP address that the OSDs shall advertise."; - example = "10.0.0.1"; - }; - - osds = mkOption { - default = { }; - example = { - osd1 = { - enable = true; - bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring"; - id = 1; - uuid = "11111111-1111-1111-1111-111111111111"; - blockDevice = "/dev/sdb"; - blockDeviceUdevRuleMatcher = ''KERNEL=="sdb"''; - clusterAddress = "10.1.0.1"; - }; - osd2 = { - enable = true; - bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring"; - id = 2; - uuid = "22222222-2222-2222-2222-222222222222"; - blockDevice = "/dev/sdc"; - blockDeviceUdevRuleMatcher = ''KERNEL=="sdc"''; - clusterAddress = "10.1.0.2"; - }; - }; - description = '' - This option allows you to define multiple Ceph OSDs. - A common idiom is to use one OSD per physical hard drive. - - Note that the OSD names given as attributes of this key - are NOT what ceph calls OSD IDs (instead, those are defined - by the 'services.ceph-benaco.osds.*.id' fields). - Instead, the name is an identifier local and unique to the - current machine only, used only to name the systemd service - for that OSD. - ''; - type = types.attrsOf ( - types.submodule { - options = { - - enable = mkEnableOption "Activate a Ceph OSD on this machine."; - - bootstrapKeyring = mkOption { - type = types.path; - description = "Ceph OSD bootstrap keyring."; - example = "/path/to/ceph.client.bootstrap-osd.keyring"; - }; - - id = mkOption { - type = types.int; - description = "The ID of this OSD. Must be unique in the Ceph cluster."; - example = 1; - }; - - uuid = mkOption { - type = types.str; - description = "The UUID of this OSD. Must be unique in the Ceph cluster."; - example = "abcdef12-abcd-1234-abcd-1234567890ab"; - }; - - systemdExtraRequiresAfter = mkOption { - type = types.listOf types.str; - default = [ ]; - description = '' - Add the specified systemd units to the "requires" and "after" - lists of the systemd service of this OSD. - - Useful, for example, to decrypt the underlying block devices with LUKS first. - - NixOS modules allow override those lists from outside, but for that - the names of the systemd services for the OSDs need to be known; - this option is a convenience to not have to know them from outside. - ''; - example = "decrypt-my-disk.service"; - }; - - skipZap = mkOption { - type = types.bool; - default = false; - description = '' - Whether to skip the zapping of the the OSD device on initial OSD - installation. - - Skipping is needed because ceph-volume cannot - zap device-mapper devices: - - - In that case you need to wipe the device manually. - - In the common case of placing the OSD on a cryptsetup LUKS device - (which is a device-mapper device), re-creating the encryption - from scratch with a new key zaps anything anyway, in which case - zapping can be skipped here. - ''; - }; - - blockDevice = mkOption { - type = types.str; - description = "The block device used to store the OSD."; - example = "/dev/sdb"; - }; - - blockDeviceUdevRuleMatcher = mkOption { - type = types.str; - description = '' - An udev rule matcher matching the block device used to store the OSD. - Will be spliced into the udev rule that is - used to set access permissions to the ceph user via an udev rule. - - This is a matcher instead of just a device name to allow flexibility: - Normal disks can be easily matched with KERNEL=="sda1", but - device-mapper may not; for example, decrypted cryptsetup LUKS devices - have a less useful KERNEL=="dm-4" and may better be matched - using ENV{DM_NAME}=="mydisk-decrypted". - ''; - example = ''KERNEL=="sdb"''; - }; - - dbBlockDevice = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - The block device used to store the OSD's BlueStore DB device. - - Put this on a faster device than to improve performance. - - See - for details. - ''; - example = "/dev/sdc"; - }; - - dbBlockDeviceUdevRuleMatcher = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Like but for the - . - ''; - example = ''KERNEL=="sdc"''; - }; - - clusterAddress = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - The IP address on the dedicated cluster network that - is used by the backend communication for OSD communication. - ''; - example = "10.1.0.1f"; - }; - - }; - } - ); - }; - - mds = { - enable = mkEnableOption "Activate a Ceph MDS on this machine."; - - nodeName = mkOption { - type = types.str; - description = "Ceph MDS node name."; - example = "node1"; - }; - - listenAddr = mkOption { - type = types.str; - description = "IP address that the MDS shall advertise."; - example = "10.0.0.1"; - }; - }; - - extraConfig = mkOption { - type = types.str; - default = ""; - description = '' - Additional ceph.conf settings. - - See the sample file for inspiration: - - ''; - }; - }; - }; - - ###### implementation - - config = - let - monDir = "/var/lib/ceph/mon/${cfg.clusterName}-${cfg.monitor.nodeName}"; - mgrDir = "/var/lib/ceph/mgr/${cfg.clusterName}-${cfg.manager.nodeName}"; - mdsDir = "/var/lib/ceph/mds/${cfg.clusterName}-${cfg.mds.nodeName}"; - - # File permissions for things that are on locations wiped at start - # (e.g. /run or its /var/run symlink). - ensureTransientCephDirs = '' - install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph - ''; - - # File permissions from cluster deployed with ceph-deploy. - ensureCephDirs = '' - install -m 3770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/log/ceph - install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph - install -m 750 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph - install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mon - install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mgr - install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/osd - ''; - - # Utilities called by Ceph device health scraping, see: - # https://docs.ceph.com/en/latest/rados/operations/devices/#enabling-monitoring - # As per https://github.com/ceph/ceph-container/pull/1490/commits/c49e821599965ae92a88b2c78077ee03c4405895, - # both the OSDs and the `mon` need this. - # Ceph calls these utilities with `sudo`. That requires sudoers entries. - # Sudoers entries require absolute path; that exact (nix store) path needs to - # be used by Ceph, so it needs to be given to the systemd unit via `path`. - # This is why we pair each `sudoersExtraRule` with the `package` to put onto - # that `path`. - # - # Entries are based on: - # https://github.com/ceph/ceph/blob/a2f5a3c1dbfa4dce41e25da4f029a8fdb8c8d864/sudoers.d/ceph-smartctl - cephMonitoringSudoersCommandsAndPackages = [ - { - package = pkgs.smartmontools; - sudoersExtraRule = { - # entry for `security.sudo.extraRules` - users = [ config.users.users.ceph.name ]; - commands = [ - { - command = "${lib.getBin pkgs.smartmontools}/bin/smartctl -x --json=o /dev/*"; - options = [ "NOPASSWD" ]; - } - ]; - }; - } - { - package = pkgs.nvme-cli; - sudoersExtraRule = { - # entry for `security.sudo.extraRules` - users = [ config.users.users.ceph.name ]; - commands = [ - { - command = "${lib.getBin pkgs.nvme-cli}/bin/nvme * smart-log-add --json /dev/*"; - options = [ "NOPASSWD" ]; - } - ]; - }; - } - ]; - - cephDeviceHealthMonitoringPathsOrPackages = - with pkgs; - [ - # Contains `sudo`. Ceph wraps this around the other health check programs. - # Cannot use `pkgs.sudo` because that one is not SUID, see: - # https://discourse.nixos.org/t/sudo-uid-issues/9133 - "/run/wrappers" # `systemd.services..path` adds the `bin/` subdir of this - ] - ++ map ({ package, ... }: package) cephMonitoringSudoersCommandsAndPackages; - - # Unused localOsdServiceName in the following line - # deadnix: skip - makeCephOsdSetupSystemdService = - _localOsdServiceName: osdConfig: - let - osdExistenceFile = "/var/lib/ceph/osd/.${toString osdConfig.id}.${osdConfig.uuid}.nix-existence"; - in - mkIf osdConfig.enable { - description = "Initialize Ceph OSD"; - - requires = osdConfig.systemdExtraRequiresAfter; - after = osdConfig.systemdExtraRequiresAfter; - - path = with pkgs; [ - # The following are currently missing in Ceph's wrapping, see https://github.com/NixOS/nixpkgs/issues/147801#issue-1065600852 - util-linux # for `lsblk` - lvm2 # for `lvs` - ]; - - # TODO Use `udevadm trigger --settle` instead of the separate `udevadm settle` - # once that feature is available to us with systemd >= 238; - # see https://github.com/systemd/systemd/commit/792cc203a67edb201073351f5c766fce3d5eab45 - preStart = '' - set -x - ${ensureCephDirs} - install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/bootstrap-osd - # `install` is not atomic, see - # https://lists.gnu.org/archive/html/bug-coreutils/2010-02/msg00243.html - # so use `mktemp` + `mv` to make it atomic. - TMPFILE=$(mktemp --tmpdir=/var/lib/ceph/bootstrap-osd/) - install -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} ${osdConfig.bootstrapKeyring} "$TMPFILE" - mv "$TMPFILE" /var/lib/ceph/bootstrap-osd/ceph.keyring - - # Trigger udev rules for permissions of block devices and wait for them to settle. - udevadm trigger --name-match=${osdConfig.blockDevice} - '' - + lib.optionalString (osdConfig.dbBlockDevice != null) '' - udevadm trigger --name-match=${osdConfig.dbBlockDevice} - '' - + '' - udevadm settle - '' - + (optionalString (!osdConfig.skipZap) ( - '' - # Zap OSD block devices, otherwise `ceph-osd` below will try to fsck if there's some old - # ceph data on the block device (see https://tracker.ceph.com/issues/24099). - ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.blockDevice} - '' - + lib.optionalString (osdConfig.dbBlockDevice != null) '' - ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.dbBlockDevice} - '' - )); - - script = '' - set -euo pipefail - set -x - until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ] - do - sleep 1 - done - - OSD_SECRET=$(${cfg.package}/bin/ceph-authtool --gen-print-key) - echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \ - ${cfg.package}/bin/ceph --cluster ${cfg.clusterName} osd new ${osdConfig.uuid} ${toString osdConfig.id} -i - \ - -n client.bootstrap-osd -k ${osdConfig.bootstrapKeyring} - mkdir -p /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id} - - ln -s ${osdConfig.blockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block - '' - + lib.optionalString (osdConfig.dbBlockDevice != null) '' - ln -s ${osdConfig.dbBlockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block.db - '' - + '' - - ${cfg.package}/bin/ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${toString osdConfig.id}/keyring \ - --name osd.${toString osdConfig.id} --add-key $OSD_SECRET - - ${cfg.package}/bin/ceph-osd -i ${toString osdConfig.id} --mkfs --osd-uuid ${osdConfig.uuid} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} --osd-objectstore bluestore - touch ${osdExistenceFile} - ''; - - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - PermissionsStartOnly = true; # only run the script as ceph, preStart as root - User = config.users.users.ceph.name; - Group = config.users.groups.ceph.name; - }; - unitConfig = { - ConditionPathExists = "!${osdExistenceFile}"; - }; - }; - - makeCephOsdSystemdService = - localOsdServiceName: osdConfig: - mkIf osdConfig.enable { - description = "Ceph OSD"; - - # Note we do not have to add `osdConfig.systemdExtraRequiresAfter` here because - # that's already a dependency of our dependency `ceph-osd-setup-*`. - requires = [ (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service") ]; - requiredBy = [ "multi-user.target" ]; - after = [ - "network.target" - "local-fs.target" - "time-sync.target" - (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service") - ]; - wants = [ - "network.target" - "local-fs.target" - "time-sync.target" - ]; - - path = [ - # TODO: use wrapProgram in the ceph package for this in the future - pkgs.getopt - ] - ++ cephDeviceHealthMonitoringPathsOrPackages; - - restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ]; - - preStart = '' - ${ensureTransientCephDirs} - ${lib.getLib cfg.package}/libexec/ceph/ceph-osd-prestart.sh --cluster ${cfg.clusterName} --id ${toString osdConfig.id} - ''; - - serviceConfig = - let - clusterIpArg = lib.optionalString ( - osdConfig.clusterAddress != null - ) "--cluster_addr=${osdConfig.clusterAddress}"; - in - { - LimitNOFILE = "1048576"; - LimitNPROC = "1048576"; - - ExecStart = '' - ${cfg.package}/bin/ceph-osd -f --cluster ${cfg.clusterName} --id ${toString osdConfig.id} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.osdBindAddr}" "--public_addr=${cfg.osdAdvertisedPublicAddr}" "${clusterIpArg}" - ''; - ExecReload = '' - ${pkgs.coreutils}/bin/kill -HUP $MAINPID - ''; - Restart = "on-failure"; - ProtectHome = "true"; - ProtectSystem = "full"; - PrivateTmp = "true"; - TasksMax = "infinity"; - # StartLimitBurst="3"; - }; - # startLimitIntervalSec = 30 * 60; - }; - - in - mkIf cfg.enable { - environment.systemPackages = [ cfg.package ]; - - networking.firewall = { - allowedTCPPorts = [ - # Ceph outside of VPN because it is very data heavy and causes packet loss. - # We enable msgr-v2 only because that allows its own on-wire encryption. - 3300 # ceph msgr-v2 - ]; - allowedTCPPortRanges = [ - { - from = 6800; - to = 7300; - } # https://docs.ceph.com/en/pacific/rados/configuration/network-config-ref/ - ]; - }; - - # Reminder of how `ceph.conf` works: - # - # * Ceph upstream docs now recommend to use underscores instead of spaces. - # * Options in more specific sections like `[mon]` override those in less - # specific sections like `[global]`. But all options can be written into all sections, - # and an option has the same name, no matter in which section it is written. - # Thus, put options in `[global]`, and only use a diffent section - # if you want to override an option you've set in `global`. - # - # Sample: https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf - environment.etc."ceph/${cfg.clusterName}.conf".text = '' - [global] - fsid = ${cfg.fsid} - mon_initial_members = ${commaSep (map (mon: mon.hostname) cfg.initialMonitors)} - mon_host = ${commaSep (map (mon: mon.ipAddress) cfg.initialMonitors)} - - # Ceph clusters go into WARN health mode, until - # the following setting is made strict by setting it to `false`: - # See: https://docs.ceph.com/en/latest/security/CVE-2021-20288/#recommendations - # As of writing, this setting is not documented outside of the CVE note :( - # - # While for new clusters the warning no longer seems to appear, it still - # appears in our existing clusters unless this option is set, see: - # https://tracker.ceph.com/issues/53751#note-7 - auth_allow_insecure_global_id_reclaim = false - - # Disable dirfrag prefetch on MDS restart to prevent out-of-memory after - # many files were opened. - # Note this option has no effect on Ceph < 15, because it doesn't exist there. - # TODO: Remove this once we're on a Ceph version that includes this default, - # see https://github.com/ceph/ceph/pull/44667. - # This is assuming that the commit fixes existing clusters, see - # https://github.com/ceph/ceph/pull/44667#issuecomment-1036103397 - # If it doesn't this can only be removed once we have no existing - # cluster with the old default. - mds_oft_prefetch_dirfrags = false - - # Disable sleep between HDD recovery operations, otherwise recovery - # will take forever when small objects (e.g. CephFS files) are on HDD. - # See https://tracker.ceph.com/issues/23595#note-12 - osd_recovery_sleep_hdd = 0.0 - - # Increase scrub intervals by 4x. - # Since we store many small files on HDD, and scrubbing apparently - # iterates over all objects - # we have no chance to scrub at the default intervals. - # - # (This was written when we had 400M files across 30 HDDs.) - # Change this back once we have reduced our number of files per disk. - osd_scrub_min_interval = 345600 - osd_scrub_max_interval = 2419200 - osd_deep_scrub_interval = 2419200 - - public_network = ${commaSep cfg.publicNetworks} - cluster_network = ${commaSep cfg.clusterNetworks} - auth_cluster_required = cephx - auth_service_required = cephx - auth_client_required = cephx - - # Enforce on-wire transport encryption. - ms_cluster_mode = secure - ms_service_mode = secure - ms_client_mode = secure - - ${cfg.extraConfig} - ''; - - environment.etc."ceph/${cfg.clusterName}.client.admin.keyring" = { - source = cfg.adminKeyring; - mode = "0600"; - # Make ceph own this keyring so that it can use it to get keys for its daemons. - user = "ceph"; - group = "ceph"; - }; - - users.users.ceph = { - isNormalUser = false; - isSystemUser = true; - # TODO: Legacy UID / GID chosen from before we configured the UID declaratively. - # In the future, we whould change this whole module to use - # `config.ids.uids.ceph`, like the upstream nixpkgs Ceph module does. - # Switching away from `nogroup` would also be good as described there. - # For both cases, we'll have to `chown` all relevant existing files on - # deployments, such as `/var/lib/ceph`, and log files. - uid = 1001; - group = config.users.groups.nogroup.name; - }; - users.groups.ceph = { - # TODO: Same TODO as above for the `uid`. - gid = 499; - }; - - # Allow ceph daemons (which run as user ceph) to collect device health metrics. - security.sudo.extraRules = map ( - { sudoersExtraRule, ... }: sudoersExtraRule - ) cephMonitoringSudoersCommandsAndPackages; - - # The udevadm trigger/settle in `makeCephOsdSetupSystemdService` waits for these rules rule to be applied. - services.udev.extraRules = lib.concatStringsSep "\n" ( - lib.mapAttrsToList ( - _localOsdServiceName: osdConfig: - '' - SUBSYSTEM=="block", ${osdConfig.blockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660" - '' - + lib.optionalString (osdConfig.dbBlockDeviceUdevRuleMatcher != null) ('' - SUBSYSTEM=="block", ${osdConfig.dbBlockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660" - '') - ) cfg.osds - ); - - systemd.services = { - - ceph-mon-setup = mkIf cfg.monitor.enable { - description = "Initialize ceph monitor"; - - preStart = ensureCephDirs; - - script = - let - # `--addv` seems currently required to get msgr-v2 working, see: - # https://tracker.ceph.com/issues/53751#note-11 - monmapNodes = builtins.concatStringsSep " " ( - lib.concatMap (mon: [ - "--addv" - mon.hostname - "[v2:${mon.ipAddress}:3300,v1:${mon.ipAddress}:6789]" - ]) cfg.initialMonitors - ); - in - # Monitors cannot simply be changed in config, one has to update the monmap, see note [replacing-ceph-monmap-ips-for-existing-cluster] - '' - set -euo pipefail - rm -rf "${monDir}" # Start from scratch. - echo "Initializing monitor." - MONMAP_DIR=`mktemp -d` - ${cfg.package}/bin/monmaptool --create ${monmapNodes} --fsid ${cfg.fsid} "$MONMAP_DIR/monmap" - ${cfg.package}/bin/ceph-mon --cluster ${cfg.clusterName} --mkfs -i ${cfg.monitor.nodeName} --monmap "$MONMAP_DIR/monmap" --keyring ${cfg.monitor.initialKeyring} - rm -r "$MONMAP_DIR" - touch ${monDir}/done - ''; - - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - PermissionsStartOnly = true; # only run the script as ceph - User = config.users.users.ceph.name; - Group = config.users.groups.ceph.name; - }; - unitConfig = { - ConditionPathExists = "!${monDir}/done"; - }; - }; - - ceph-mon = mkIf cfg.monitor.enable { - description = "Ceph monitor"; - - requires = [ (ensureUnitExists config "ceph-mon-setup.service") ]; - requiredBy = [ "multi-user.target" ]; - after = [ - "network.target" - "local-fs.target" - "time-sync.target" - (ensureUnitExists config "ceph-mon-setup.service") - ]; - wants = [ - "network.target" - "local-fs.target" - "time-sync.target" - ]; - - restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ]; - - path = cephDeviceHealthMonitoringPathsOrPackages; - - preStart = ensureTransientCephDirs; - - serviceConfig = { - LimitNOFILE = "1048576"; - LimitNPROC = "1048576"; - ExecStart = '' - ${cfg.package}/bin/ceph-mon -f --cluster ${cfg.clusterName} --id ${cfg.monitor.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.monitor.bindAddr}" "--public_addr=${cfg.monitor.advertisedPublicAddr}" - ''; - ExecReload = '' - ${pkgs.coreutils}/bin/kill -HUP $MAINPID - ''; - PrivateDevices = "yes"; - ProtectHome = "true"; - ProtectSystem = "full"; - PrivateTmp = "true"; - TasksMax = "infinity"; - Restart = "on-failure"; - # StartLimitBurst="5"; - RestartSec = "10"; - }; - # startLimitIntervalSec = 30 * 60; - }; - - ceph-mgr-setup = mkIf cfg.manager.enable { - description = "Initialize Ceph manager"; - - preStart = ensureCephDirs; - - script = '' - set -euo pipefail - mkdir -p ${mgrDir} - until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ] - do - sleep 1 - done - ${cfg.package}/bin/ceph auth get-or-create mgr.${cfg.manager.nodeName} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o ${mgrDir}/keyring - touch "${mgrDir}/.nix_done" - ''; - - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - PermissionsStartOnly = true; # only run the script as ceph - User = config.users.users.ceph.name; - Group = config.users.groups.ceph.name; - }; - unitConfig = { - ConditionPathExists = "!${mgrDir}/.nix_done"; - }; - }; - - ceph-mgr = mkIf cfg.manager.enable { - description = "Ceph manager"; - - requires = [ (ensureUnitExists config "ceph-mgr-setup.service") ]; - requiredBy = [ "multi-user.target" ]; - after = [ - "network.target" - "local-fs.target" - "time-sync.target" - (ensureUnitExists config "ceph-mgr-setup.service") - ]; - wants = [ - "network.target" - "local-fs.target" - "time-sync.target" - ]; - - restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ]; - - preStart = ensureTransientCephDirs; - - serviceConfig = { - LimitNOFILE = "1048576"; - LimitNPROC = "1048576"; - - ExecStart = '' - ${cfg.package}/bin/ceph-mgr -f --cluster ${cfg.clusterName} --id ${cfg.manager.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} - ''; - ExecReload = '' - ${pkgs.coreutils}/bin/kill -HUP $MAINPID - ''; - Restart = "on-failure"; - RestartSec = 10; - # StartLimitBurst="3"; - }; - # startLimitIntervalSec = 30 * 60; - }; - - ceph-mds-setup = mkIf cfg.mds.enable { - description = "Initialize Ceph MDS"; - - preStart = ensureCephDirs; - - script = '' - set -euo pipefail - mkdir -p ${mdsDir} - until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ] - do - sleep 1 - done - ${cfg.package}/bin/ceph auth get-or-create mds.${cfg.mds.nodeName} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o ${mdsDir}/keyring - touch "${mdsDir}/.nix_done" - ''; - - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - PermissionsStartOnly = true; # only run the script as ceph - User = config.users.users.ceph.name; - Group = config.users.groups.ceph.name; - }; - unitConfig = { - ConditionPathExists = "!${mdsDir}/.nix_done"; - }; - }; - - ceph-mds = mkIf cfg.mds.enable { - description = "Ceph MDS"; - - requires = [ (ensureUnitExists config "ceph-mds-setup.service") ]; - requiredBy = [ "multi-user.target" ]; - after = [ - "network.target" - "local-fs.target" - "time-sync.target" - (ensureUnitExists config "ceph-mds-setup.service") - ]; - wants = [ - "network.target" - "local-fs.target" - "time-sync.target" - ]; - - restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ]; - - preStart = ensureTransientCephDirs; - - serviceConfig = { - LimitNOFILE = "1048576"; - LimitNPROC = "1048576"; - - ExecStart = '' - ${cfg.package}/bin/ceph-mds -f --cluster ${cfg.clusterName} --id ${cfg.mds.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_addr=${cfg.mds.listenAddr}" - ''; - ExecReload = '' - ${pkgs.coreutils}/bin/kill -HUP $MAINPID - ''; - Restart = "on-failure"; - # StartLimitBurst="3"; - }; - # startLimitIntervalSec = 30 * 60; - }; - - } - # Make one OSD service for each configured OSD. - // lib.mapAttrs' ( - localOsdServiceName: osdConfig: - nameValuePair "ceph-osd-setup-${localOsdServiceName}" ( - makeCephOsdSetupSystemdService localOsdServiceName osdConfig - ) - ) cfg.osds - // lib.mapAttrs' ( - localOsdServiceName: osdConfig: - nameValuePair "ceph-osd-${localOsdServiceName}" ( - makeCephOsdSystemdService localOsdServiceName osdConfig - ) - ) cfg.osds; - }; -} diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index a0676c5..a29a08c 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -7,7 +7,6 @@ ./adblock-update.nix ./albyhub.nix ./backup.nix - ./ceph.nix ./db.nix ./gitea-runner.nix ./gnome.nix diff --git a/network.json b/network.json index 3bcec47..137e4b0 100644 --- a/network.json +++ b/network.json @@ -9,10 +9,6 @@ "tailscale": "100.64.0.0/10" }, "hosts": { - "builder2": { - "ip": "10.42.1.17", - "system": "x86_64-linux" - }, "exodus": { "ip": null, "pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3", diff --git a/secrets/home.client.admin.keyring b/secrets/home.client.admin.keyring deleted file mode 100644 index 6b40c04..0000000 --- a/secrets/home.client.admin.keyring +++ /dev/null @@ -1,6 +0,0 @@ -[client.admin] - key = AQBojDlmfnc8MBAAkr+PXbSewmq4OooESo2X1A== - caps mds = "allow *" - caps mgr = "allow *" - caps mon = "allow *" - caps osd = "allow *" diff --git a/secrets/home.keyring b/secrets/home.keyring deleted file mode 100644 index e69de29..0000000 diff --git a/secrets/home.mon.keyring b/secrets/home.mon.keyring deleted file mode 100644 index 87c51fe..0000000 --- a/secrets/home.mon.keyring +++ /dev/null @@ -1,13 +0,0 @@ -[mon.] - key = AQAUijlm7emnJBAAKsHT1+2EzYRQxKsL4KwwkQ== - caps mon = "allow *" -[client.admin] - key = AQBojDlmfnc8MBAAkr+PXbSewmq4OooESo2X1A== - caps mds = "allow *" - caps mgr = "allow *" - caps mon = "allow *" - caps osd = "allow *" -[client.bootstrap-osd] - key = AQDvJDpm4BDlIhAAXISJWnrOtNDk0FqhSX0/YQ== - caps mgr = "allow r" - caps mon = "profile bootstrap-osd" diff --git a/secrets/home.osd-bootstrap.keyring b/secrets/home.osd-bootstrap.keyring deleted file mode 100644 index 2990cb2..0000000 --- a/secrets/home.osd-bootstrap.keyring +++ /dev/null @@ -1,4 +0,0 @@ -[client.bootstrap-osd] - key = AQDvJDpm4BDlIhAAXISJWnrOtNDk0FqhSX0/YQ== - caps mgr = "allow r" - caps mon = "profile bootstrap-osd"