diff --git a/darwin/hosts/MacBook-Prolocal b/darwin/hosts/MacBook-Prolocal
index c12c63e..f6cb244 120000
--- a/darwin/hosts/MacBook-Prolocal
+++ b/darwin/hosts/MacBook-Prolocal
@@ -1 +1 @@
-gregory
\ No newline at end of file
+ivr
\ No newline at end of file
diff --git a/darwin/hosts/gregory b/darwin/hosts/gregory
new file mode 120000
index 0000000..f6cb244
--- /dev/null
+++ b/darwin/hosts/gregory
@@ -0,0 +1 @@
+ivr
\ No newline at end of file
diff --git a/darwin/hosts/gregory.hellings-mbp b/darwin/hosts/gregory.hellings-mbp
index c12c63e..f6cb244 120000
--- a/darwin/hosts/gregory.hellings-mbp
+++ b/darwin/hosts/gregory.hellings-mbp
@@ -1 +1 @@
-gregory
\ No newline at end of file
+ivr
\ No newline at end of file
diff --git a/darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024 b/darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024
new file mode 120000
index 0000000..45a5001
--- /dev/null
+++ b/darwin/hosts/gregs-MacBook-Pro-16-inch-Nov-2024
@@ -0,0 +1 @@
+lithic
\ No newline at end of file
diff --git a/darwin/hosts/ivr b/darwin/hosts/ivr
deleted file mode 120000
index ed27f59..0000000
--- a/darwin/hosts/ivr
+++ /dev/null
@@ -1 +0,0 @@
-gregory/
\ No newline at end of file
diff --git a/darwin/hosts/gregory/default.nix b/darwin/hosts/ivr/default.nix
similarity index 100%
rename from darwin/hosts/gregory/default.nix
rename to darwin/hosts/ivr/default.nix
diff --git a/home/hosts/builder2/default.nix b/home/hosts/builder2/default.nix
deleted file mode 100644
index 6462967..0000000
--- a/home/hosts/builder2/default.nix
+++ /dev/null
@@ -1 +0,0 @@
-{...}: {}
diff --git a/modules/nixos/ceph.nix b/modules/nixos/ceph.nix
deleted file mode 100644
index 7bc7676..0000000
--- a/modules/nixos/ceph.nix
+++ /dev/null
@@ -1,927 +0,0 @@
-# This is a good source for a Ceph dealio
-# https://gist.github.com0/nh2/13425a1f18b4c1ce82edb63c10b163c9
-{
- config,
- lib,
- pkgs,
- ...
-}:
-
-with lib;
-
-let
- cfg = config.services.ceph-benaco;
- commaSep = builtins.concatStringsSep ",";
-
- ensureUnitExists =
- c': name:
- let
- in
- #unitName = (builtins.elemAt (builtins.split "\\." name) 0);
- if c'.systemd.services ? unitName then name else name; # "Unable to locate ${name} at ${commaSep (builtins.attrNames c')}";
-in
-
-{
-
- ###### interface
-
- options = {
-
- services.ceph-benaco = {
-
- enable = mkEnableOption "Ceph distributed filesystem";
-
- package = mkOption {
- type = types.package;
- default = pkgs.ceph;
- defaultText = literalExpression "pkgs.ceph-benaco";
- description = "Ceph package to use.";
- };
-
- fsid = mkOption {
- type = types.str;
- description = "Unique cluster identifier.";
- };
-
- clusterName = mkOption {
- type = types.str;
- description = "Cluster name.";
- default = "ceph";
- };
-
- initialMonitors = mkOption {
- type = types.listOf (
- types.submodule {
- options = {
- hostname = mkOption {
- type = types.str;
- description = "Initial monitor hostname.";
- };
-
- ipAddress = mkOption {
- type = types.str;
- description = "Initial monitor IP address.";
- };
- };
- }
- );
- description = "Initial monitors.";
- };
-
- mdsNodes = mkOption {
- type = types.listOf (
- types.submodule {
- options = {
- hostname = mkOption {
- type = types.str;
- description = "MDS hostname.";
- };
-
- ipAddress = mkOption {
- type = types.str;
- description = "MDS IP address.";
- };
- };
- }
- );
- description = "MDS nodes.";
- };
-
- publicNetworks = mkOption {
- type = types.listOf types.str;
- description = "Public network(s) of the cluster.";
- };
-
- clusterNetworks = mkOption {
- type = types.listOf types.str;
- description = "Cluster backend networks for OSD sync";
- };
-
- adminKeyring = mkOption {
- type = types.path;
- description = "Ceph admin keyring to install on the machine.";
- };
-
- monitor = {
- enable = mkEnableOption "Activate a Ceph monitor on this machine.";
-
- initialKeyring = mkOption {
- type = types.path;
- description = "Keyring file to use when initializing a new monitor";
- example = "/path/to/ceph.mon.keyring";
- };
-
- nodeName = mkOption {
- type = types.str;
- description = "Ceph monitor node name.";
- example = "node1";
- };
-
- bindAddr = mkOption {
- type = types.str;
- description = "IP address that the OSDs shall bind to.";
- example = "10.0.0.1";
- };
-
- advertisedPublicAddr = mkOption {
- type = types.str;
- description = "IP address that the monitor shall advertise.";
- example = "10.0.0.1";
- };
- };
-
- manager = {
- enable = mkEnableOption "Activate a Ceph manager on this machine.";
-
- nodeName = mkOption {
- type = types.str;
- description = "Ceph manager node name.";
- example = "node1";
- };
- };
-
- osdBindAddr = mkOption {
- type = types.str;
- description = "IP address that the OSDs shall bind to.";
- example = "10.0.0.1";
- };
-
- osdAdvertisedPublicAddr = mkOption {
- type = types.str;
- description = "IP address that the OSDs shall advertise.";
- example = "10.0.0.1";
- };
-
- osds = mkOption {
- default = { };
- example = {
- osd1 = {
- enable = true;
- bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
- id = 1;
- uuid = "11111111-1111-1111-1111-111111111111";
- blockDevice = "/dev/sdb";
- blockDeviceUdevRuleMatcher = ''KERNEL=="sdb"'';
- clusterAddress = "10.1.0.1";
- };
- osd2 = {
- enable = true;
- bootstrapKeyring = "/path/to/ceph.client.bootstrap-osd.keyring";
- id = 2;
- uuid = "22222222-2222-2222-2222-222222222222";
- blockDevice = "/dev/sdc";
- blockDeviceUdevRuleMatcher = ''KERNEL=="sdc"'';
- clusterAddress = "10.1.0.2";
- };
- };
- description = ''
- This option allows you to define multiple Ceph OSDs.
- A common idiom is to use one OSD per physical hard drive.
-
- Note that the OSD names given as attributes of this key
- are NOT what ceph calls OSD IDs (instead, those are defined
- by the 'services.ceph-benaco.osds.*.id' fields).
- Instead, the name is an identifier local and unique to the
- current machine only, used only to name the systemd service
- for that OSD.
- '';
- type = types.attrsOf (
- types.submodule {
- options = {
-
- enable = mkEnableOption "Activate a Ceph OSD on this machine.";
-
- bootstrapKeyring = mkOption {
- type = types.path;
- description = "Ceph OSD bootstrap keyring.";
- example = "/path/to/ceph.client.bootstrap-osd.keyring";
- };
-
- id = mkOption {
- type = types.int;
- description = "The ID of this OSD. Must be unique in the Ceph cluster.";
- example = 1;
- };
-
- uuid = mkOption {
- type = types.str;
- description = "The UUID of this OSD. Must be unique in the Ceph cluster.";
- example = "abcdef12-abcd-1234-abcd-1234567890ab";
- };
-
- systemdExtraRequiresAfter = mkOption {
- type = types.listOf types.str;
- default = [ ];
- description = ''
- Add the specified systemd units to the "requires" and "after"
- lists of the systemd service of this OSD.
-
- Useful, for example, to decrypt the underlying block devices with LUKS first.
-
- NixOS modules allow override those lists from outside, but for that
- the names of the systemd services for the OSDs need to be known;
- this option is a convenience to not have to know them from outside.
- '';
- example = "decrypt-my-disk.service";
- };
-
- skipZap = mkOption {
- type = types.bool;
- default = false;
- description = ''
- Whether to skip the zapping of the the OSD device on initial OSD
- installation.
-
- Skipping is needed because ceph-volume cannot
- zap device-mapper devices:
-
-
- In that case you need to wipe the device manually.
-
- In the common case of placing the OSD on a cryptsetup LUKS device
- (which is a device-mapper device), re-creating the encryption
- from scratch with a new key zaps anything anyway, in which case
- zapping can be skipped here.
- '';
- };
-
- blockDevice = mkOption {
- type = types.str;
- description = "The block device used to store the OSD.";
- example = "/dev/sdb";
- };
-
- blockDeviceUdevRuleMatcher = mkOption {
- type = types.str;
- description = ''
- An udev rule matcher matching the block device used to store the OSD.
- Will be spliced into the udev rule that is
- used to set access permissions to the ceph user via an udev rule.
-
- This is a matcher instead of just a device name to allow flexibility:
- Normal disks can be easily matched with KERNEL=="sda1", but
- device-mapper may not; for example, decrypted cryptsetup LUKS devices
- have a less useful KERNEL=="dm-4" and may better be matched
- using ENV{DM_NAME}=="mydisk-decrypted".
- '';
- example = ''KERNEL=="sdb"'';
- };
-
- dbBlockDevice = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- The block device used to store the OSD's BlueStore DB device.
-
- Put this on a faster device than to improve performance.
-
- See
- for details.
- '';
- example = "/dev/sdc";
- };
-
- dbBlockDeviceUdevRuleMatcher = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- Like but for the
- .
- '';
- example = ''KERNEL=="sdc"'';
- };
-
- clusterAddress = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- The IP address on the dedicated cluster network that
- is used by the backend communication for OSD communication.
- '';
- example = "10.1.0.1f";
- };
-
- };
- }
- );
- };
-
- mds = {
- enable = mkEnableOption "Activate a Ceph MDS on this machine.";
-
- nodeName = mkOption {
- type = types.str;
- description = "Ceph MDS node name.";
- example = "node1";
- };
-
- listenAddr = mkOption {
- type = types.str;
- description = "IP address that the MDS shall advertise.";
- example = "10.0.0.1";
- };
- };
-
- extraConfig = mkOption {
- type = types.str;
- default = "";
- description = ''
- Additional ceph.conf settings.
-
- See the sample file for inspiration:
-
- '';
- };
- };
- };
-
- ###### implementation
-
- config =
- let
- monDir = "/var/lib/ceph/mon/${cfg.clusterName}-${cfg.monitor.nodeName}";
- mgrDir = "/var/lib/ceph/mgr/${cfg.clusterName}-${cfg.manager.nodeName}";
- mdsDir = "/var/lib/ceph/mds/${cfg.clusterName}-${cfg.mds.nodeName}";
-
- # File permissions for things that are on locations wiped at start
- # (e.g. /run or its /var/run symlink).
- ensureTransientCephDirs = ''
- install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
- '';
-
- # File permissions from cluster deployed with ceph-deploy.
- ensureCephDirs = ''
- install -m 3770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/log/ceph
- install -m 770 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/run/ceph
- install -m 750 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph
- install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mon
- install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/mgr
- install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/osd
- '';
-
- # Utilities called by Ceph device health scraping, see:
- # https://docs.ceph.com/en/latest/rados/operations/devices/#enabling-monitoring
- # As per https://github.com/ceph/ceph-container/pull/1490/commits/c49e821599965ae92a88b2c78077ee03c4405895,
- # both the OSDs and the `mon` need this.
- # Ceph calls these utilities with `sudo`. That requires sudoers entries.
- # Sudoers entries require absolute path; that exact (nix store) path needs to
- # be used by Ceph, so it needs to be given to the systemd unit via `path`.
- # This is why we pair each `sudoersExtraRule` with the `package` to put onto
- # that `path`.
- #
- # Entries are based on:
- # https://github.com/ceph/ceph/blob/a2f5a3c1dbfa4dce41e25da4f029a8fdb8c8d864/sudoers.d/ceph-smartctl
- cephMonitoringSudoersCommandsAndPackages = [
- {
- package = pkgs.smartmontools;
- sudoersExtraRule = {
- # entry for `security.sudo.extraRules`
- users = [ config.users.users.ceph.name ];
- commands = [
- {
- command = "${lib.getBin pkgs.smartmontools}/bin/smartctl -x --json=o /dev/*";
- options = [ "NOPASSWD" ];
- }
- ];
- };
- }
- {
- package = pkgs.nvme-cli;
- sudoersExtraRule = {
- # entry for `security.sudo.extraRules`
- users = [ config.users.users.ceph.name ];
- commands = [
- {
- command = "${lib.getBin pkgs.nvme-cli}/bin/nvme * smart-log-add --json /dev/*";
- options = [ "NOPASSWD" ];
- }
- ];
- };
- }
- ];
-
- cephDeviceHealthMonitoringPathsOrPackages =
- with pkgs;
- [
- # Contains `sudo`. Ceph wraps this around the other health check programs.
- # Cannot use `pkgs.sudo` because that one is not SUID, see:
- # https://discourse.nixos.org/t/sudo-uid-issues/9133
- "/run/wrappers" # `systemd.services..path` adds the `bin/` subdir of this
- ]
- ++ map ({ package, ... }: package) cephMonitoringSudoersCommandsAndPackages;
-
- # Unused localOsdServiceName in the following line
- # deadnix: skip
- makeCephOsdSetupSystemdService =
- _localOsdServiceName: osdConfig:
- let
- osdExistenceFile = "/var/lib/ceph/osd/.${toString osdConfig.id}.${osdConfig.uuid}.nix-existence";
- in
- mkIf osdConfig.enable {
- description = "Initialize Ceph OSD";
-
- requires = osdConfig.systemdExtraRequiresAfter;
- after = osdConfig.systemdExtraRequiresAfter;
-
- path = with pkgs; [
- # The following are currently missing in Ceph's wrapping, see https://github.com/NixOS/nixpkgs/issues/147801#issue-1065600852
- util-linux # for `lsblk`
- lvm2 # for `lvs`
- ];
-
- # TODO Use `udevadm trigger --settle` instead of the separate `udevadm settle`
- # once that feature is available to us with systemd >= 238;
- # see https://github.com/systemd/systemd/commit/792cc203a67edb201073351f5c766fce3d5eab45
- preStart = ''
- set -x
- ${ensureCephDirs}
- install -m 755 -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} -d /var/lib/ceph/bootstrap-osd
- # `install` is not atomic, see
- # https://lists.gnu.org/archive/html/bug-coreutils/2010-02/msg00243.html
- # so use `mktemp` + `mv` to make it atomic.
- TMPFILE=$(mktemp --tmpdir=/var/lib/ceph/bootstrap-osd/)
- install -o ${config.users.users.ceph.name} -g ${config.users.groups.ceph.name} ${osdConfig.bootstrapKeyring} "$TMPFILE"
- mv "$TMPFILE" /var/lib/ceph/bootstrap-osd/ceph.keyring
-
- # Trigger udev rules for permissions of block devices and wait for them to settle.
- udevadm trigger --name-match=${osdConfig.blockDevice}
- ''
- + lib.optionalString (osdConfig.dbBlockDevice != null) ''
- udevadm trigger --name-match=${osdConfig.dbBlockDevice}
- ''
- + ''
- udevadm settle
- ''
- + (optionalString (!osdConfig.skipZap) (
- ''
- # Zap OSD block devices, otherwise `ceph-osd` below will try to fsck if there's some old
- # ceph data on the block device (see https://tracker.ceph.com/issues/24099).
- ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.blockDevice}
- ''
- + lib.optionalString (osdConfig.dbBlockDevice != null) ''
- ${cfg.package}/bin/ceph-volume lvm zap ${osdConfig.dbBlockDevice}
- ''
- ));
-
- script = ''
- set -euo pipefail
- set -x
- until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
- do
- sleep 1
- done
-
- OSD_SECRET=$(${cfg.package}/bin/ceph-authtool --gen-print-key)
- echo "{\"cephx_secret\": \"$OSD_SECRET\"}" | \
- ${cfg.package}/bin/ceph --cluster ${cfg.clusterName} osd new ${osdConfig.uuid} ${toString osdConfig.id} -i - \
- -n client.bootstrap-osd -k ${osdConfig.bootstrapKeyring}
- mkdir -p /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}
-
- ln -s ${osdConfig.blockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block
- ''
- + lib.optionalString (osdConfig.dbBlockDevice != null) ''
- ln -s ${osdConfig.dbBlockDevice} /var/lib/ceph/osd/${cfg.clusterName}-${toString osdConfig.id}/block.db
- ''
- + ''
-
- ${cfg.package}/bin/ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${toString osdConfig.id}/keyring \
- --name osd.${toString osdConfig.id} --add-key $OSD_SECRET
-
- ${cfg.package}/bin/ceph-osd -i ${toString osdConfig.id} --mkfs --osd-uuid ${osdConfig.uuid} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} --osd-objectstore bluestore
- touch ${osdExistenceFile}
- '';
-
- serviceConfig = {
- Type = "oneshot";
- RemainAfterExit = true;
- PermissionsStartOnly = true; # only run the script as ceph, preStart as root
- User = config.users.users.ceph.name;
- Group = config.users.groups.ceph.name;
- };
- unitConfig = {
- ConditionPathExists = "!${osdExistenceFile}";
- };
- };
-
- makeCephOsdSystemdService =
- localOsdServiceName: osdConfig:
- mkIf osdConfig.enable {
- description = "Ceph OSD";
-
- # Note we do not have to add `osdConfig.systemdExtraRequiresAfter` here because
- # that's already a dependency of our dependency `ceph-osd-setup-*`.
- requires = [ (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service") ];
- requiredBy = [ "multi-user.target" ];
- after = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- (ensureUnitExists config "ceph-osd-setup-${localOsdServiceName}.service")
- ];
- wants = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- ];
-
- path = [
- # TODO: use wrapProgram in the ceph package for this in the future
- pkgs.getopt
- ]
- ++ cephDeviceHealthMonitoringPathsOrPackages;
-
- restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
- preStart = ''
- ${ensureTransientCephDirs}
- ${lib.getLib cfg.package}/libexec/ceph/ceph-osd-prestart.sh --cluster ${cfg.clusterName} --id ${toString osdConfig.id}
- '';
-
- serviceConfig =
- let
- clusterIpArg = lib.optionalString (
- osdConfig.clusterAddress != null
- ) "--cluster_addr=${osdConfig.clusterAddress}";
- in
- {
- LimitNOFILE = "1048576";
- LimitNPROC = "1048576";
-
- ExecStart = ''
- ${cfg.package}/bin/ceph-osd -f --cluster ${cfg.clusterName} --id ${toString osdConfig.id} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.osdBindAddr}" "--public_addr=${cfg.osdAdvertisedPublicAddr}" "${clusterIpArg}"
- '';
- ExecReload = ''
- ${pkgs.coreutils}/bin/kill -HUP $MAINPID
- '';
- Restart = "on-failure";
- ProtectHome = "true";
- ProtectSystem = "full";
- PrivateTmp = "true";
- TasksMax = "infinity";
- # StartLimitBurst="3";
- };
- # startLimitIntervalSec = 30 * 60;
- };
-
- in
- mkIf cfg.enable {
- environment.systemPackages = [ cfg.package ];
-
- networking.firewall = {
- allowedTCPPorts = [
- # Ceph outside of VPN because it is very data heavy and causes packet loss.
- # We enable msgr-v2 only because that allows its own on-wire encryption.
- 3300 # ceph msgr-v2
- ];
- allowedTCPPortRanges = [
- {
- from = 6800;
- to = 7300;
- } # https://docs.ceph.com/en/pacific/rados/configuration/network-config-ref/
- ];
- };
-
- # Reminder of how `ceph.conf` works:
- #
- # * Ceph upstream docs now recommend to use underscores instead of spaces.
- # * Options in more specific sections like `[mon]` override those in less
- # specific sections like `[global]`. But all options can be written into all sections,
- # and an option has the same name, no matter in which section it is written.
- # Thus, put options in `[global]`, and only use a diffent section
- # if you want to override an option you've set in `global`.
- #
- # Sample: https://github.com/ceph/ceph/blob/master/src/sample.ceph.conf
- environment.etc."ceph/${cfg.clusterName}.conf".text = ''
- [global]
- fsid = ${cfg.fsid}
- mon_initial_members = ${commaSep (map (mon: mon.hostname) cfg.initialMonitors)}
- mon_host = ${commaSep (map (mon: mon.ipAddress) cfg.initialMonitors)}
-
- # Ceph clusters go into WARN health mode, until
- # the following setting is made strict by setting it to `false`:
- # See: https://docs.ceph.com/en/latest/security/CVE-2021-20288/#recommendations
- # As of writing, this setting is not documented outside of the CVE note :(
- #
- # While for new clusters the warning no longer seems to appear, it still
- # appears in our existing clusters unless this option is set, see:
- # https://tracker.ceph.com/issues/53751#note-7
- auth_allow_insecure_global_id_reclaim = false
-
- # Disable dirfrag prefetch on MDS restart to prevent out-of-memory after
- # many files were opened.
- # Note this option has no effect on Ceph < 15, because it doesn't exist there.
- # TODO: Remove this once we're on a Ceph version that includes this default,
- # see https://github.com/ceph/ceph/pull/44667.
- # This is assuming that the commit fixes existing clusters, see
- # https://github.com/ceph/ceph/pull/44667#issuecomment-1036103397
- # If it doesn't this can only be removed once we have no existing
- # cluster with the old default.
- mds_oft_prefetch_dirfrags = false
-
- # Disable sleep between HDD recovery operations, otherwise recovery
- # will take forever when small objects (e.g. CephFS files) are on HDD.
- # See https://tracker.ceph.com/issues/23595#note-12
- osd_recovery_sleep_hdd = 0.0
-
- # Increase scrub intervals by 4x.
- # Since we store many small files on HDD, and scrubbing apparently
- # iterates over all objects
- # we have no chance to scrub at the default intervals.
- #
- # (This was written when we had 400M files across 30 HDDs.)
- # Change this back once we have reduced our number of files per disk.
- osd_scrub_min_interval = 345600
- osd_scrub_max_interval = 2419200
- osd_deep_scrub_interval = 2419200
-
- public_network = ${commaSep cfg.publicNetworks}
- cluster_network = ${commaSep cfg.clusterNetworks}
- auth_cluster_required = cephx
- auth_service_required = cephx
- auth_client_required = cephx
-
- # Enforce on-wire transport encryption.
- ms_cluster_mode = secure
- ms_service_mode = secure
- ms_client_mode = secure
-
- ${cfg.extraConfig}
- '';
-
- environment.etc."ceph/${cfg.clusterName}.client.admin.keyring" = {
- source = cfg.adminKeyring;
- mode = "0600";
- # Make ceph own this keyring so that it can use it to get keys for its daemons.
- user = "ceph";
- group = "ceph";
- };
-
- users.users.ceph = {
- isNormalUser = false;
- isSystemUser = true;
- # TODO: Legacy UID / GID chosen from before we configured the UID declaratively.
- # In the future, we whould change this whole module to use
- # `config.ids.uids.ceph`, like the upstream nixpkgs Ceph module does.
- # Switching away from `nogroup` would also be good as described there.
- # For both cases, we'll have to `chown` all relevant existing files on
- # deployments, such as `/var/lib/ceph`, and log files.
- uid = 1001;
- group = config.users.groups.nogroup.name;
- };
- users.groups.ceph = {
- # TODO: Same TODO as above for the `uid`.
- gid = 499;
- };
-
- # Allow ceph daemons (which run as user ceph) to collect device health metrics.
- security.sudo.extraRules = map (
- { sudoersExtraRule, ... }: sudoersExtraRule
- ) cephMonitoringSudoersCommandsAndPackages;
-
- # The udevadm trigger/settle in `makeCephOsdSetupSystemdService` waits for these rules rule to be applied.
- services.udev.extraRules = lib.concatStringsSep "\n" (
- lib.mapAttrsToList (
- _localOsdServiceName: osdConfig:
- ''
- SUBSYSTEM=="block", ${osdConfig.blockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
- ''
- + lib.optionalString (osdConfig.dbBlockDeviceUdevRuleMatcher != null) (''
- SUBSYSTEM=="block", ${osdConfig.dbBlockDeviceUdevRuleMatcher}, OWNER="${config.users.users.ceph.name}", GROUP="${config.users.groups.ceph.name}", MODE="0660"
- '')
- ) cfg.osds
- );
-
- systemd.services = {
-
- ceph-mon-setup = mkIf cfg.monitor.enable {
- description = "Initialize ceph monitor";
-
- preStart = ensureCephDirs;
-
- script =
- let
- # `--addv` seems currently required to get msgr-v2 working, see:
- # https://tracker.ceph.com/issues/53751#note-11
- monmapNodes = builtins.concatStringsSep " " (
- lib.concatMap (mon: [
- "--addv"
- mon.hostname
- "[v2:${mon.ipAddress}:3300,v1:${mon.ipAddress}:6789]"
- ]) cfg.initialMonitors
- );
- in
- # Monitors cannot simply be changed in config, one has to update the monmap, see note [replacing-ceph-monmap-ips-for-existing-cluster]
- ''
- set -euo pipefail
- rm -rf "${monDir}" # Start from scratch.
- echo "Initializing monitor."
- MONMAP_DIR=`mktemp -d`
- ${cfg.package}/bin/monmaptool --create ${monmapNodes} --fsid ${cfg.fsid} "$MONMAP_DIR/monmap"
- ${cfg.package}/bin/ceph-mon --cluster ${cfg.clusterName} --mkfs -i ${cfg.monitor.nodeName} --monmap "$MONMAP_DIR/monmap" --keyring ${cfg.monitor.initialKeyring}
- rm -r "$MONMAP_DIR"
- touch ${monDir}/done
- '';
-
- serviceConfig = {
- Type = "oneshot";
- RemainAfterExit = true;
- PermissionsStartOnly = true; # only run the script as ceph
- User = config.users.users.ceph.name;
- Group = config.users.groups.ceph.name;
- };
- unitConfig = {
- ConditionPathExists = "!${monDir}/done";
- };
- };
-
- ceph-mon = mkIf cfg.monitor.enable {
- description = "Ceph monitor";
-
- requires = [ (ensureUnitExists config "ceph-mon-setup.service") ];
- requiredBy = [ "multi-user.target" ];
- after = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- (ensureUnitExists config "ceph-mon-setup.service")
- ];
- wants = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- ];
-
- restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
- path = cephDeviceHealthMonitoringPathsOrPackages;
-
- preStart = ensureTransientCephDirs;
-
- serviceConfig = {
- LimitNOFILE = "1048576";
- LimitNPROC = "1048576";
- ExecStart = ''
- ${cfg.package}/bin/ceph-mon -f --cluster ${cfg.clusterName} --id ${cfg.monitor.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_bind_addr=${cfg.monitor.bindAddr}" "--public_addr=${cfg.monitor.advertisedPublicAddr}"
- '';
- ExecReload = ''
- ${pkgs.coreutils}/bin/kill -HUP $MAINPID
- '';
- PrivateDevices = "yes";
- ProtectHome = "true";
- ProtectSystem = "full";
- PrivateTmp = "true";
- TasksMax = "infinity";
- Restart = "on-failure";
- # StartLimitBurst="5";
- RestartSec = "10";
- };
- # startLimitIntervalSec = 30 * 60;
- };
-
- ceph-mgr-setup = mkIf cfg.manager.enable {
- description = "Initialize Ceph manager";
-
- preStart = ensureCephDirs;
-
- script = ''
- set -euo pipefail
- mkdir -p ${mgrDir}
- until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
- do
- sleep 1
- done
- ${cfg.package}/bin/ceph auth get-or-create mgr.${cfg.manager.nodeName} mon 'allow profile mgr' mds 'allow *' osd 'allow *' -o ${mgrDir}/keyring
- touch "${mgrDir}/.nix_done"
- '';
-
- serviceConfig = {
- Type = "oneshot";
- RemainAfterExit = true;
- PermissionsStartOnly = true; # only run the script as ceph
- User = config.users.users.ceph.name;
- Group = config.users.groups.ceph.name;
- };
- unitConfig = {
- ConditionPathExists = "!${mgrDir}/.nix_done";
- };
- };
-
- ceph-mgr = mkIf cfg.manager.enable {
- description = "Ceph manager";
-
- requires = [ (ensureUnitExists config "ceph-mgr-setup.service") ];
- requiredBy = [ "multi-user.target" ];
- after = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- (ensureUnitExists config "ceph-mgr-setup.service")
- ];
- wants = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- ];
-
- restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
- preStart = ensureTransientCephDirs;
-
- serviceConfig = {
- LimitNOFILE = "1048576";
- LimitNPROC = "1048576";
-
- ExecStart = ''
- ${cfg.package}/bin/ceph-mgr -f --cluster ${cfg.clusterName} --id ${cfg.manager.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name}
- '';
- ExecReload = ''
- ${pkgs.coreutils}/bin/kill -HUP $MAINPID
- '';
- Restart = "on-failure";
- RestartSec = 10;
- # StartLimitBurst="3";
- };
- # startLimitIntervalSec = 30 * 60;
- };
-
- ceph-mds-setup = mkIf cfg.mds.enable {
- description = "Initialize Ceph MDS";
-
- preStart = ensureCephDirs;
-
- script = ''
- set -euo pipefail
- mkdir -p ${mdsDir}
- until [ -f /etc/ceph/${cfg.clusterName}.client.admin.keyring ]
- do
- sleep 1
- done
- ${cfg.package}/bin/ceph auth get-or-create mds.${cfg.mds.nodeName} osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o ${mdsDir}/keyring
- touch "${mdsDir}/.nix_done"
- '';
-
- serviceConfig = {
- Type = "oneshot";
- RemainAfterExit = true;
- PermissionsStartOnly = true; # only run the script as ceph
- User = config.users.users.ceph.name;
- Group = config.users.groups.ceph.name;
- };
- unitConfig = {
- ConditionPathExists = "!${mdsDir}/.nix_done";
- };
- };
-
- ceph-mds = mkIf cfg.mds.enable {
- description = "Ceph MDS";
-
- requires = [ (ensureUnitExists config "ceph-mds-setup.service") ];
- requiredBy = [ "multi-user.target" ];
- after = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- (ensureUnitExists config "ceph-mds-setup.service")
- ];
- wants = [
- "network.target"
- "local-fs.target"
- "time-sync.target"
- ];
-
- restartTriggers = [ config.environment.etc."ceph/${cfg.clusterName}.conf".source ];
-
- preStart = ensureTransientCephDirs;
-
- serviceConfig = {
- LimitNOFILE = "1048576";
- LimitNPROC = "1048576";
-
- ExecStart = ''
- ${cfg.package}/bin/ceph-mds -f --cluster ${cfg.clusterName} --id ${cfg.mds.nodeName} --setuser ${config.users.users.ceph.name} --setgroup ${config.users.groups.ceph.name} "--public_addr=${cfg.mds.listenAddr}"
- '';
- ExecReload = ''
- ${pkgs.coreutils}/bin/kill -HUP $MAINPID
- '';
- Restart = "on-failure";
- # StartLimitBurst="3";
- };
- # startLimitIntervalSec = 30 * 60;
- };
-
- }
- # Make one OSD service for each configured OSD.
- // lib.mapAttrs' (
- localOsdServiceName: osdConfig:
- nameValuePair "ceph-osd-setup-${localOsdServiceName}" (
- makeCephOsdSetupSystemdService localOsdServiceName osdConfig
- )
- ) cfg.osds
- // lib.mapAttrs' (
- localOsdServiceName: osdConfig:
- nameValuePair "ceph-osd-${localOsdServiceName}" (
- makeCephOsdSystemdService localOsdServiceName osdConfig
- )
- ) cfg.osds;
- };
-}
diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index febe84f..78d8785 100644
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -7,7 +7,6 @@
./adblock-update.nix
./albyhub.nix
./backup.nix
- ./ceph.nix
./db.nix
./gitea-runner.nix
./gnome.nix
diff --git a/network.json b/network.json
index 3bcec47..137e4b0 100644
--- a/network.json
+++ b/network.json
@@ -9,10 +9,6 @@
"tailscale": "100.64.0.0/10"
},
"hosts": {
- "builder2": {
- "ip": "10.42.1.17",
- "system": "x86_64-linux"
- },
"exodus": {
"ip": null,
"pubkey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxmnCj2E9DxcnefPW+n4yCuLShxqr0p024riogdeXA3",
diff --git a/secrets/home.client.admin.keyring b/secrets/home.client.admin.keyring
deleted file mode 100644
index 6b40c04..0000000
--- a/secrets/home.client.admin.keyring
+++ /dev/null
@@ -1,6 +0,0 @@
-[client.admin]
- key = AQBojDlmfnc8MBAAkr+PXbSewmq4OooESo2X1A==
- caps mds = "allow *"
- caps mgr = "allow *"
- caps mon = "allow *"
- caps osd = "allow *"
diff --git a/secrets/home.keyring b/secrets/home.keyring
deleted file mode 100644
index e69de29..0000000
diff --git a/secrets/home.mon.keyring b/secrets/home.mon.keyring
deleted file mode 100644
index 87c51fe..0000000
--- a/secrets/home.mon.keyring
+++ /dev/null
@@ -1,13 +0,0 @@
-[mon.]
- key = AQAUijlm7emnJBAAKsHT1+2EzYRQxKsL4KwwkQ==
- caps mon = "allow *"
-[client.admin]
- key = AQBojDlmfnc8MBAAkr+PXbSewmq4OooESo2X1A==
- caps mds = "allow *"
- caps mgr = "allow *"
- caps mon = "allow *"
- caps osd = "allow *"
-[client.bootstrap-osd]
- key = AQDvJDpm4BDlIhAAXISJWnrOtNDk0FqhSX0/YQ==
- caps mgr = "allow r"
- caps mon = "profile bootstrap-osd"
diff --git a/secrets/home.osd-bootstrap.keyring b/secrets/home.osd-bootstrap.keyring
deleted file mode 100644
index 2990cb2..0000000
--- a/secrets/home.osd-bootstrap.keyring
+++ /dev/null
@@ -1,4 +0,0 @@
-[client.bootstrap-osd]
- key = AQDvJDpm4BDlIhAAXISJWnrOtNDk0FqhSX0/YQ==
- caps mgr = "allow r"
- caps mon = "profile bootstrap-osd"