Add vfio stuff for win10

Add a whole bunch of stuff to start/stop displays for the win10 vm
Hopefully this will allow win10 to directly use the same GPU in passthru
mode

hosts/jude/virt.nix

@@ -27,7 +27,7 @@ in
       "vfio"
       "vfio_iommu_type1"
 
-      "amdgpu"
+      #"amdgpu"
       #"nvidia"
       #"nvidia_modeset"
       #"nvidia_uvm"
@@ -35,6 +35,7 @@ in
     ];
     kernelParams = [
       "amd_iommu=on"
+      "iommu=pt"
       ("vfio-pci.ids=" + (lib.concatStringsSep "," passthru))
     ];
   };
@@ -64,12 +65,28 @@ in
       wantedBy = pkgs.lib.mkForce [ ];
       serviceConfig.User = "root";
     };
+
+    "libvirtd-nosleep@" = {
+      description = "Prevent sleep while %i is running";
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = ''
+          ${pkgs.systemd}/bin/systemd-inhibit --what=sleep --why="Libvirt domain %i is running" --who=%U --mode=block sleep infinity
+        '';
+      };
+    };
   };
 
   virtualisation = {
-    libvirtd.hooks.qemu =
+    libvirtd = {
-      {
+      extraConfig = ''
+        log_filters="1:qemu"
+        log_outputs="1:file:/var/log/libvirt/libvirtd.log"
+      '';
+      hooks.qemu = {
+        win10 = lib.getExe pkgs.qemu-hook;
       };
+    };
 
     spiceUSBRedirection.enable = true;
   };

modules/nixos/vmdev.nix

@@ -34,7 +34,9 @@ with lib;
       OVMFFull
       nixos-generators
       packer
+      swtpm
       virt-manager
+      virtio-win
       xorriso
     ];
 
@@ -45,11 +47,16 @@ with lib;
       libvirtd = {
         enable = true;
         onBoot = "ignore"; # Do not auto-restart VMs on boot, unless they are marked autostart
-        qemu.ovmf = {
+        qemu = {
-          enable = true;
+          ovmf = {
-          packages = [ pkgs.OVMFFull.fd ];
+            enable = true;
+            packages = [ pkgs.OVMFFull.fd ];
+          };
+          swtpm = {
+            enable = true;
+            package = pkgs.swtpm;
+          };
         };
-        qemu.swtpm.enable = true;
       };
 
       virtualbox.host = {

overlays/default.nix

@@ -39,6 +39,7 @@ rec {
   # My own packages
   enwiki-dump = prev.callPackage ./enwiki-dump.nix { };
 
+  qemu = prev.qemu.override { sdlSupport = true; };
   # Overrides of packages
   libbluray-custom = prev.libbluray.override {
     withAACS = true;

pkgs/default.nix

@@ -13,7 +13,10 @@ in
   inject-darwin = c ./inject-darwin.nix { };
   inject = c ./inject.nix { };
   hms = c ./hms { };
+  qemu-hook = c ./qemu-hook.nix { };
   setup-ssh = c ./setup-ssh { };
   upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
+  vfio_startup = c ./vfio_startup.nix { };
+  vfio_shutdown = c ./vfio_shutdown.nix { };
   zim = c ./zim.nix { };
 }

pkgs/qemu-hook.nix

@@ -0,0 +1,31 @@
+{
+  writeShellApplication,
+  vfio_shutdown,
+  vfio_startup,
+}:
+
+writeShellApplication {
+  name = "qemu_hook";
+  runtimeInputs = [
+    vfio_shutdown
+    vfio_startup
+  ];
+
+  text = ''
+    OBJECT="''${1}"
+    OPERATION="''${2}"
+
+    if [[ "''${OBJECT}" == "win10" ]]; then
+      case "''${OPERATION}" in
+        "prepare")
+          systemctl start libvirt-nosleep@"''${OBJECT}".service
+          systemctl start bluetooth.service
+          vfio_startup 2>&1 | tee -a /var/log/qemu_hook.log
+          ;;
+        "release")
+          systemctl stop libvirt-nosleep@"''${OBJECT}".service
+          vfio_shutdown 2>&1 | tee -a /var/log/qemu_hook.log
+      esac
+    fi
+  '';
+}

pkgs/vfio_shutdown.nix

@@ -0,0 +1,32 @@
+{
+  writeShellApplication,
+  coreutils,
+  kmod,
+  systemd,
+}:
+
+writeShellApplication {
+  name = "vfio_shutdown";
+  runtimeInputs = [
+    coreutils
+    kmod
+    systemd
+  ];
+  text = ''
+    echo "Removing vfio modules..."
+    modprobe -r vfio_pci
+    modprobe -r vfio_iommu_type1
+    modprobe -r vfio
+
+    echo "Reload the GPU modules..."
+    modprobe nvidia
+    modprobe nvidia_modeset
+    modprobe nvidia_drm
+    modprobe nvidia_uvm
+
+    echo "Starting the display manager..."
+    systemctl start display-manager.service
+
+    echo "All done!"
+  '';
+}

pkgs/vfio_startup.nix

@@ -0,0 +1,33 @@
+{
+  writeShellApplication,
+  kmod,
+  systemd,
+}:
+
+writeShellApplication {
+  name = "vfio_startup";
+  runtimeInputs = [
+    kmod
+    systemd
+  ];
+  text = ''
+    # Stop the service that uses the GPU
+    echo "Stopping the display manager..."
+    systemctl stop display-manager.service
+
+    # Unload the GPU modules
+    echo "Unloading the GPU modules..."
+    modprobe -r nvidia_uvm
+    modprobe -r nvidia_drm
+    modprobe -r nvidia_modeset
+    modprobe -r nvidia
+
+    # Insert the vfio modules
+    echo "Inserting the vfio modules..."
+    modprobe vfio
+    modprobe vfio_pci
+    modprobe vfio_iommu_type1
+
+    echo "Aaaand... off you go!"
+  '';
+}