KVM/libvirtd kinda working

2025-01-07 10:58:24 -06:00
parent 2029daffc2
commit 8a0b0f55e8
5 changed files with 113 additions and 146 deletions
@@ -115,11 +115,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1728330715,
-        "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
+        "lastModified": 1735644329,
+        "narHash": "sha256-tO3HrHriyLvipc4xr+Ewtdlo7wM1OjXNjlWRgmM7peY=",
        "owner": "numtide",
        "repo": "devshell",
-        "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
+        "rev": "f7795ede5b02664b57035b3b757876703e2c3eac",
        "type": "github"
      },
      "original": {
@@ -257,11 +257,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1733312601,
-        "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=",
+        "lastModified": 1736143030,
+        "narHash": "sha256-+hu54pAoLDEZT9pjHlqL9DNzWz0NbUn8NEAHP7PQPzU=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
-        "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9",
+        "rev": "b905f6fc23a9051a6e1b741e1438dbfc0634c6de",
        "type": "github"
      },
      "original": {
@@ -345,24 +345,6 @@
        "type": "github"
      }
    },
-    "flake-utils_4": {
-      "inputs": {
-        "systems": "systems_6"
-      },
-      "locked": {
-        "lastModified": 1731533236,
-        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
    "git-hooks": {
      "inputs": {
        "flake-compat": [
@@ -373,18 +355,14 @@
        "nixpkgs": [
          "nixvimunstable",
          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nixvimunstable",
-          "nixpkgs"
        ]
      },
      "locked": {
-        "lastModified": 1734425854,
-        "narHash": "sha256-nzE5UbJ41aPEKf8R2ZFYtLkqPmF7EIUbNEdHMBLg0Ig=",
+        "lastModified": 1735882644,
+        "narHash": "sha256-3FZAG+pGt3OElQjesCAWeMkQ7C/nB1oTHLRQ8ceP110=",
        "owner": "cachix",
        "repo": "git-hooks.nix",
-        "rev": "0ddd26d0925f618c3a5d85a4fa5eb1e23a09491d",
+        "rev": "a5a961387e75ae44cc20f0a57ae463da5e959656",
        "type": "github"
      },
      "original": {
@@ -443,11 +421,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1734344598,
-        "narHash": "sha256-wNX3hsScqDdqKWOO87wETUEi7a/QlPVgpC/Lh5rFOuA=",
+        "lastModified": 1736204492,
+        "narHash": "sha256-CoBPRgkUex9Iz6qGSzi/BFVUQjndB0PmME2B6eEyeCs=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "83ecd50915a09dca928971139d3a102377a8d242",
+        "rev": "20665c6efa83d71020c8730f26706258ba5c6b2a",
        "type": "github"
      },
      "original": {
@@ -486,11 +464,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1734344598,
-        "narHash": "sha256-wNX3hsScqDdqKWOO87wETUEi7a/QlPVgpC/Lh5rFOuA=",
+        "lastModified": 1736089250,
+        "narHash": "sha256-/LPWMiiJGPHGd7ZYEgmbE2da4zvBW0acmshUjYC3WG4=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "83ecd50915a09dca928971139d3a102377a8d242",
+        "rev": "172b91bfb2b7f5c4a8c6ceac29fd53a01ef07196",
        "type": "github"
      },
      "original": {
@@ -556,11 +534,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1733570843,
-        "narHash": "sha256-sQJAxY1TYWD1UyibN/FnN97paTFuwBw3Vp3DNCyKsMk=",
+        "lastModified": 1736085891,
+        "narHash": "sha256-bTl9fcUo767VaSx4Q5kFhwiDpFQhBKna7lNbGsqCQiA=",
        "owner": "lnl7",
        "repo": "nix-darwin",
-        "rev": "a35b08d09efda83625bef267eb24347b446c80b8",
+        "rev": "ba9b3173b0f642ada42b78fb9dfc37ca82266f6c",
        "type": "github"
      },
      "original": {
@@ -729,11 +707,11 @@
    },
    "nixpkgs_3": {
      "locked": {
-        "lastModified": 1734424634,
-        "narHash": "sha256-cHar1vqHOOyC7f1+tVycPoWTfKIaqkoe1Q6TnKzuti4=",
+        "lastModified": 1736012469,
+        "narHash": "sha256-/qlNWm/IEVVH7GfgAIyP6EsVZI6zjAx1cV5zNyrs+rI=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "d3c42f187194c26d9f0309a8ecc469d6c878ce33",
+        "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d",
        "type": "github"
      },
      "original": {
@@ -745,11 +723,11 @@
    },
    "nixpkgs_4": {
      "locked": {
-        "lastModified": 1734424634,
-        "narHash": "sha256-cHar1vqHOOyC7f1+tVycPoWTfKIaqkoe1Q6TnKzuti4=",
+        "lastModified": 1736012469,
+        "narHash": "sha256-/qlNWm/IEVVH7GfgAIyP6EsVZI6zjAx1cV5zNyrs+rI=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "d3c42f187194c26d9f0309a8ecc469d6c878ce33",
+        "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d",
        "type": "github"
      },
      "original": {
@@ -793,11 +771,11 @@
    },
    "nixunstable": {
      "locked": {
-        "lastModified": 1734424634,
-        "narHash": "sha256-cHar1vqHOOyC7f1+tVycPoWTfKIaqkoe1Q6TnKzuti4=",
+        "lastModified": 1736012469,
+        "narHash": "sha256-/qlNWm/IEVVH7GfgAIyP6EsVZI6zjAx1cV5zNyrs+rI=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "d3c42f187194c26d9f0309a8ecc469d6c878ce33",
+        "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d",
        "type": "github"
      },
      "original": {
@@ -820,11 +798,11 @@
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
-        "lastModified": 1734566698,
-        "narHash": "sha256-6UTHBkxdI+Bn67T+NFN4tGeeww5bOfUUP1OQnluWTtQ=",
+        "lastModified": 1736219611,
+        "narHash": "sha256-osQC3uZOQQrTPTlhEBE6mesPCkYDDOSLsJNNATQZqmw=",
        "owner": "nix-community",
        "repo": "nixvim",
-        "rev": "2b7f17b6de578b63b7dc0e4155e36538042bc968",
+        "rev": "a352bb89b0cd180f5c767689c4cc310109cd4ad8",
        "type": "github"
      },
      "original": {
@@ -841,11 +819,11 @@
        "treefmt-nix": "treefmt-nix_2"
      },
      "locked": {
-        "lastModified": 1734561640,
-        "narHash": "sha256-67IMskLnI513MgkJxo/Sn/mDcwx7R+VuQvmi+lzSkZY=",
+        "lastModified": 1736263064,
+        "narHash": "sha256-UGc/ikH9GUWfErzTXcyttdqC18ih/NCtL+vO6pOkiFk=",
        "owner": "nix-community",
        "repo": "NUR",
-        "rev": "9371cbccda4136da7cbfb93711eb1cf4fb97d0af",
+        "rev": "ca9c757ffce0193240967cf5d485758bea1b4f05",
        "type": "github"
      },
      "original": {
@@ -864,11 +842,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1733773348,
-        "narHash": "sha256-Y47y+LesOCkJaLvj+dI/Oa6FAKj/T9sKVKDXLNsViPw=",
+        "lastModified": 1735854821,
+        "narHash": "sha256-Iv59gMDZajNfezTO0Fw6LHE7uKAShxbvMidmZREit7c=",
        "owner": "NuschtOS",
        "repo": "search",
-        "rev": "3051be7f403bff1d1d380e4612f0c70675b44fc9",
+        "rev": "836908e3bddd837ae0f13e215dd48767aee355f0",
        "type": "github"
      },
      "original": {
@@ -993,21 +971,6 @@
        "type": "github"
      }
    },
-    "systems_6": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
    "treefmt-nix": {
      "inputs": {
        "nixpkgs": [
@@ -1016,11 +979,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1733761991,
-        "narHash": "sha256-s4DalCDepD22jtKL5Nw6f4LP5UwoMcPzPZgHWjAfqbQ=",
+        "lastModified": 1736115332,
+        "narHash": "sha256-FBG9d7e0BTFfxVdw4b5EmNll2Mv7hfRc54hbB4LrKko=",
        "owner": "numtide",
        "repo": "treefmt-nix",
-        "rev": "0ce9d149d99bc383d1f2d85f31f6ebd146e46085",
+        "rev": "1788ca5acd4b542b923d4757d4cfe4183cc6a92d",
        "type": "github"
      },
      "original": {
@@ -1075,11 +1038,11 @@
        "nixpkgs": "nixpkgs_6"
      },
      "locked": {
-        "lastModified": 1734486856,
-        "narHash": "sha256-+DI4DiwcS3Zp1OUuEi38Tb5gpVt+c8eDpQHy6MAHPzo=",
+        "lastModified": 1736214462,
+        "narHash": "sha256-JcwPAIcZ3DeNimp+tULr83MBSTTZIfQBwvuhw5wMVt0=",
        "owner": "nix-community",
        "repo": "nix-vscode-extensions",
-        "rev": "94a50a7dc9b4be928f36544d1538c6bae93f6f94",
+        "rev": "9893dc8ad11e92b461e75a8ae51473c8620aa0a8",
        "type": "github"
      },
      "original": {
@@ -1091,17 +1054,16 @@
    "wsl": {
      "inputs": {
        "flake-compat": "flake-compat_5",
-        "flake-utils": "flake-utils_4",
        "nixpkgs": [
          "nixunstable"
        ]
      },
      "locked": {
-        "lastModified": 1733854371,
-        "narHash": "sha256-K9qGHniYBbjqVcEiwXyiofj/IFf78L5F0/FCf+CKyr0=",
+        "lastModified": 1736095716,
+        "narHash": "sha256-csysw/Szu98QDiA2lhWk9seYOyCebeVEWL89zh1cduM=",
        "owner": "nix-community",
        "repo": "NixOS-WSL",
-        "rev": "dee4425dcee3149475ead0cb6a616b8a028c5888",
+        "rev": "63c3b4ed1712a3a0621002cd59bfdc80875ecbb0",
        "type": "github"
      },
      "original": {
@@ -12,7 +12,11 @@

  home = {
    file.".config/libvirt/qemu.conf".text = ''
-      nvram = [ "/run/libvirt/nix-ovmf/OVMF_CODE.fd:/run/libvirt/nix-ovmf/OVMF_VARS.fd" ]
+      nvram = [
+        "/run/libvirt/nix-ovmf/OVMF_CODE.fd:/run/libvirt/nix-ovmf/OVMF_VARS.fd",
+        "/run/libvirt/nix-ovmf/AAVMF_CODE.fd:/run/libvirt/nix-ovmf/AAVMF_VARS.fd",
+        "${pkgs.OVMFFull.fd}/FV/OVMF_CODE.ms.fd:${pkgs.OVMFFull.fd}/FV/OVMF_VARS.ms.fd"
+      ]
    '';

    packages = with pkgs; [
@@ -1,23 +1,45 @@
-{ pkgs, config, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:

+let
+  passthru = [
+    "1002:164e" # Raphael - embedded GPU
+    "1002:1640" # Rembrandt - Audio
+    #"10de:2507" # RTX 3050 video
+    #"10de:228e" # RTX 3050 audio
+  ];
+in
 {
  greg.vmdev.enable = true;

-  systemd.services = {
-    gitlab-runner = {
-      conflicts = [ "libvirtd.service" ];
-      preStart = builtins.concatStringsSep "\n" [
-        "${pkgs.kmod}/bin/modprobe vboxnetflt vboxdrv"
-        "${pkgs.kmod}/bin/modprobe vboxnetadp"
-      ];
-      postStop = "${pkgs.kmod}/bin/rmmod vboxnetflt vboxnetadp vboxdrv";
-      wantedBy = pkgs.lib.mkForce [ ];
-      serviceConfig.User = "root";
-    };
-  };
-
  age.secrets.runner-reg.file = ../../secrets/gitlab/isaiah-vbox-runner-reg.age;

+  # These options enable sharing of the GPU with the VM
+  boot = {
+    # Order matters here, to prevent the AMD driver from getting to the driver before
+    # vfio-pci does
+    initrd.kernelModules = [
+      "vfio_pci"
+      "vfio"
+      "vfio_iommu_type1"
+
+      "amdgpu"
+      #"nvidia"
+      #"nvidia_modeset"
+      #"nvidia_uvm"
+      #"nvidia_drm"
+    ];
+    kernelParams = [
+      "amd_iommu=on"
+      ("vfio-pci.ids=" + (lib.concatStringsSep "," passthru))
+    ];
+  };
+  hardware.opengl.enable = true;
+
  services.gitlab-runner = {
    enable = true;
    settings.concurrent = 5;
@@ -31,53 +53,24 @@
    };
  };

-  virtualisation = {
-    libvirtd.hooks.qemu = {
-      "win11-prepare-begin-bind_vfio.sh" = pkgs.writeShellApplication {
-        name = "bind_vfio.sh";
-        runtimeInputs = with pkgs; [
-          kmod
-          libvirt
-        ];
-        text = ''
-          guest="$1"
-          stage="$2"
-          state="$3"
-
-          if [ "$guest" == "win11" ] && [ "$stage" == "prepare" ] && [ "$state" == "begin" ]; then
-            for f in vfio vfio_iommu_type1 vfio_pci; do
-              modprobe "$f"
-            done
-
-            virsh nodedev-detach pci_0000_10_00_0
-            virsh nodedev-detach pci_0000_10_00_1
-          fi
-        '';
-      };
-
-      "win11-release-end-unbind_vfio.sh" = pkgs.writeShellApplication {
-        name = "unbind_vfio.sh";
-        runtimeInputs = with pkgs; [
-          kmod
-          libvirt
-        ];
-        text = ''
-          guest="$1"
-          stage="$2"
-          state="$3"
-
-          if [ "$guest" == "win11" ] && [ "$stage" == "release" ] && [ "$state" == "end" ]; then
-            virsh nodedev-reattach pci_0000_10_00_1
-            virsh nodedev-reattach pci_0000_10_00_0
-
-            for f in vfio_pci vfio_iommu_type1 vfio; do
-              modprobe -r "$f"
-            done
-          fi
-        '';
-      };
+  systemd.services = {
+    gitlab-runner = {
+      conflicts = [ "libvirtd.service" ];
+      preStart = builtins.concatStringsSep "\n" [
+        "${pkgs.kmod}/bin/modprobe vboxnetflt vboxdrv"
+        "${pkgs.kmod}/bin/modprobe vboxnetadp"
+      ];
+      postStop = "${pkgs.kmod}/bin/rmmod vboxnetflt vboxnetadp vboxdrv";
+      wantedBy = pkgs.lib.mkForce [ ];
+      serviceConfig.User = "root";
    };
-    lxd.enable = false;
-    waydroid.enable = false;
+  };
+
+  virtualisation = {
+    libvirtd.hooks.qemu =
+      {
+      };
+
+    spiceUSBRedirection.enable = true;
  };
 }
@@ -45,7 +45,10 @@ with lib;
      libvirtd = {
        enable = true;
        onBoot = "ignore"; # Do not auto-restart VMs on boot, unless they are marked autostart
-        qemu.ovmf.enable = true;
+        qemu.ovmf = {
+          enable = true;
+          packages = [ pkgs.OVMFFull.fd ];
+        };
        qemu.swtpm.enable = true;
      };

@@ -46,4 +46,9 @@ rec {
  };
  handbrake = prev.handbrake.override { libbluray = libbluray-custom; };
  pipenv-ivr = prev.callPackage ./pipenv.nix { };
+  OVMFFull = prev.OVMFFull.override {
+    secureBoot = true;
+    msVarsTemplate = true;
+    tpmSupport = true;
+  };
 }