From 1afe353df7b5987c5c2defcfc6287e333ecf37ad Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sun, 17 Dec 2023 15:47:52 -0600 Subject: [PATCH] Bring up the VM runners --- hosts/myself/container-runner-qemu.nix | 63 +++++++++++ hosts/myself/container-runner-vbox.nix | 84 ++++++++++++++ hosts/myself/default.nix | 10 -- hosts/myself/git.nix | 103 ++++++++++-------- ...r-reg.age => myself-qemu-runner-reg-1.age} | 0 secrets/gitlab/myself-qemu-runner-reg-2.age | 23 ++++ secrets/gitlab/myself-qemu-runner-reg-3.age | Bin 0 -> 1117 bytes secrets/gitlab/myself-qemu-runner-reg-4.age | Bin 0 -> 1241 bytes secrets/gitlab/myself-qemu-runner-reg-5.age | 21 ++++ secrets/gitlab/myself-vbox-runner-reg.age | 22 ++++ secrets/secrets.nix | 7 +- 11 files changed, 274 insertions(+), 59 deletions(-) create mode 100644 hosts/myself/container-runner-qemu.nix create mode 100644 hosts/myself/container-runner-vbox.nix rename secrets/gitlab/{myself-qemu-runner-reg.age => myself-qemu-runner-reg-1.age} (100%) create mode 100644 secrets/gitlab/myself-qemu-runner-reg-2.age create mode 100644 secrets/gitlab/myself-qemu-runner-reg-3.age create mode 100644 secrets/gitlab/myself-qemu-runner-reg-4.age create mode 100644 secrets/gitlab/myself-qemu-runner-reg-5.age create mode 100644 secrets/gitlab/myself-vbox-runner-reg.age diff --git a/hosts/myself/container-runner-qemu.nix b/hosts/myself/container-runner-qemu.nix new file mode 100644 index 0000000..551d1da --- /dev/null +++ b/hosts/myself/container-runner-qemu.nix @@ -0,0 +1,63 @@ +inputs: +{ config, pkgs, ... }: +let + extraPackages = with pkgs; [ + curl + gawk + git + packer + pup + (python3.withPackages (p: with p; [ pip pyyaml virtualenv ])) + qemu_full + qemu_kvm + shellcheck + xonsh + xorriso + ]; +in { + imports = [ + inputs.agenix.nixosModules.default + ]; + age.identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + age.secrets = let + file = num: { + file = ../../secrets/gitlab/myself-qemu-runner-reg-${num}.age; + owner = "gitlab-runner"; + }; + in { + qemu-runner-reg-1 = file "1"; + qemu-runner-reg-2 = file "2"; + qemu-runner-reg-3 = file "3"; + qemu-runner-reg-4 = file "4"; + qemu-runner-reg-5 = file "5"; + }; + + networking.useHostResolvConf = pkgs.lib.mkForce false; + networking.nameservers = [ "100.100.100.100" ]; + services.resolved.enable = true; + + environment.systemPackages = extraPackages; + + services.gitlab-runner = { + enable = true; + settings.concurrent = 5; + services = let + r = num: { + executor = "shell"; + registrationConfigFile = config.age.secrets."qemu-runner-reg-${num}".path; + tagList = [ "shell" "qemu" ]; + }; + in { + shell1 = r "1"; + shell2 = r "2"; + shell3 = r "3"; + shell4 = r "4"; + shell5 = r "5"; + }; + }; + + systemd.services.gitlab-runner.wants = [ "network-online.target" ]; + systemd.services.gitlab-runner.after = [ "network.target" "network-online.target" ]; + + system.stateVersion = "24.05"; +} diff --git a/hosts/myself/container-runner-vbox.nix b/hosts/myself/container-runner-vbox.nix new file mode 100644 index 0000000..93c46f7 --- /dev/null +++ b/hosts/myself/container-runner-vbox.nix @@ -0,0 +1,84 @@ +{ inputs, name, extra}: + +({ config, pkgs, lib, ... }: +let + py = (pkgs.python3.withPackages (p: with p; [ + pip + pyyaml + virtualenv + ])); +in ( + lib.attrsets.recursiveUpdate { + + imports = [ + inputs.agenix.nixosModules.default + ]; + + age = { + identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + secrets.runner-reg = { + file = ../../secrets/gitlab/myself-${name}-runner-reg.age; + owner = "gitlab-runner"; + }; + }; + + environment.systemPackages = with pkgs; [ + curl + gawk + git + packer + pup + py + shellcheck + xorriso + ]; + + networking = { + useHostResolvConf = pkgs.lib.mkForce false; + nameservers = [ "100.100.100.100" ]; + }; + + nixpkgs.config.allowUnfree = true; + + services = { + gitlab-runner = { + enable = true; + settings.concurrent = 5; + services = { + shell = { + executor = "shell"; + limit = 5; + registrationConfigFile = config.age.secrets.runner-reg.path; + tagList = [ "shell" name ]; + }; + }; + }; + resolved.enable = true; + }; + + systemd.services.gitlab-runner = { + wants = [ "network-online.target" ]; + after = [ "network.target" "network-online.target" ]; + serviceConfig = { + User = "root"; + DynamicUser = lib.mkForce false; + }; + }; + + system.stateVersion = "24.05"; + users.users.gitlab-runner = { + isNormalUser = true; + group = "gitlab-runner"; + extraGroups = [ + "root" + "sudo" + "vboxusers" + "wheel" + ]; + }; + users.groups.gitlab-runner = {}; + } + + extra +) # End of attrsets.recursiveUpdate +) # End of outter function wrapper diff --git a/hosts/myself/default.nix b/hosts/myself/default.nix index 48c2c8d..040759f 100644 --- a/hosts/myself/default.nix +++ b/hosts/myself/default.nix @@ -36,16 +36,6 @@ "10.42.1.5" ]; }; - virtualisation = { - libvirtd = { - enable = false; - onBoot = "ignore"; - }; - virtualbox.host = { - enable = true; - enableExtensionPack = true; - }; - }; users = { users = { greg = { diff --git a/hosts/myself/git.nix b/hosts/myself/git.nix index 94d85ab..90a97fe 100644 --- a/hosts/myself/git.nix +++ b/hosts/myself/git.nix @@ -1,24 +1,15 @@ { config, pkgs, lib, inputs, ... }: let - extraPackages = with pkgs; [ - config.virtualisation.virtualbox.host.package - curl - gawk - git - packer - pup - (python3.withPackages (p: with p; [ pip virtualenv ])) - qemu_full - qemu_kvm - shellcheck - xonsh - xorriso - ]; gitlabStateDir = "/var/lib/gitlab"; registryPort = 8001; + + container = input: (lib.attrsets.recursiveUpdate { + bindMounts."/etc/ssh".hostPath = "/etc/ssh"; # For agenix secrets + privateNetwork = true; + } input); in { networking = { firewall = { @@ -36,33 +27,36 @@ in { system.activationScripts.makeGitlabDir = lib.stringAfter [ "var" ] "mkdir -p ${gitlabStateDir} && touch ${gitlabStateDir}/touch"; - containers.gitlab = { + containers.gitlab = container { autoStart = true; bindMounts = { "/var/gitlab/state" = { hostPath = gitlabStateDir; isReadOnly = false; }; - "/etc/ssh".hostPath = "/etc/ssh"; }; forwardPorts = [{ hostPort = 2222; containerPort = 22; }]; - privateNetwork = true; hostAddress = "192.168.200.1"; localAddress = "192.168.200.2"; config = ((import ./container-git.nix) { inherit inputs registryPort; }); }; - systemd.services."container@gitlab-runner".serviceConfig = { + systemd.services."container@gitlab-runner-qemu".serviceConfig = { DevicePolicy = lib.mkForce "auto"; + ExecPostStop = [ + "rmmod kvm_amd kvm" + ]; + ExecPreStart = [ + "modprobe kvm" + ]; }; + systemd.services."container@gitlab-runner-qemu".conflicts = [ "container@gitlab-runner-vbox.service" ]; - containers.gitlab-runner = { - autoStart = true; + containers.gitlab-runner-qemu = container { bindMounts = { - "/etc/ssh".hostPath = "/etc/ssh"; "/dev/kvm" = { hostPath = "/dev/kvm"; isReadOnly = false; @@ -71,37 +65,50 @@ in { extraFlags = [ "--property=DeviceAllow=/dev/kvm" ]; - privateNetwork = true; hostAddress = "192.168.201.1"; localAddress = "192.168.201.2"; - config = { config, pkgs, ... }: { - imports = [ - inputs.agenix.nixosModules.default - ]; - age.identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; - age.secrets.qemu-runner-reg = { - file = ../../secrets/gitlab/myself-qemu-runner-reg.age; - owner = "gitlab-runner"; + config = ((import ./container-runner-qemu.nix) inputs); + }; + + systemd.services."container@gitlab-runner-vbox".serviceConfig = { + DevicePolicy = lib.mkForce "auto"; + ExecPostStop = [ + "rmmod vboxnetadp vboxnetflt vboxdrv" + ]; + ExecPreStart = [ + "modprobe vboxdrv vboxnetadp vboxnetflt" + ]; + }; + systemd.services."container@gitlab-runner-vbox".conflicts = [ "container@gitlab-runner-qemu.service" ]; + + containers.gitlab-runner-vbox = container { + bindMounts = { + "/dev/vboxdrv" = { + hostPath = "/dev/vboxdrv"; + isReadOnly = false; }; - - networking.useHostResolvConf = lib.mkForce false; - networking.nameservers = [ "100.100.100.100" ]; - services.resolved.enable = true; - - environment.systemPackages = extraPackages; - - services.gitlab-runner = { - enable = true; - services = { - shell = { - executor = "shell"; - limit = 5; - registrationConfigFile = config.age.secrets.qemu-runner-reg.path; - tagList = [ "shell" "qemu" ]; - }; + "/dev/vboxdrvu" = { + hostPath = "/dev/vboxdrvu"; + isReadOnly = false; + }; + "/dev/vboxnetctl" = { + hostPath = "/dev/vboxnetctl"; + isReadOnly = false; + }; + }; + hostAddress = "192.168.202.1"; + localAddress = "192.168.202.2"; + config = ((import ./container-runner-vbox.nix) { + inherit inputs; + name = "vbox"; + extra = { + virtualisation.virtualbox.host = { + enable = true; + enableExtensionPack = true; + enableHardening = false; + headless = true; }; }; - system.stateVersion = "24.05"; - }; + }); }; } diff --git a/secrets/gitlab/myself-qemu-runner-reg.age b/secrets/gitlab/myself-qemu-runner-reg-1.age similarity index 100% rename from secrets/gitlab/myself-qemu-runner-reg.age rename to secrets/gitlab/myself-qemu-runner-reg-1.age diff --git a/secrets/gitlab/myself-qemu-runner-reg-2.age b/secrets/gitlab/myself-qemu-runner-reg-2.age new file mode 100644 index 0000000..e2d7314 --- /dev/null +++ b/secrets/gitlab/myself-qemu-runner-reg-2.age @@ -0,0 +1,23 @@ +age-encryption.org/v1 +-> ssh-ed25519 mOmPfg QRKinSglUT5mjchw4/mDO91hAUm8oEXGrUkrQTjcPic +/NB/ucCSX6dNmHYLO4HMxucilwt7hxTHeUIKDc8kuh0 +-> ssh-ed25519 YJiRbw yE7YWVJ3jgW7sPlygMTezfkeS6TAjRdGxdV53K5JOVY +QdwrGrM0oUyIN2KXFFTmw+wcbLASirz54wLtrsyTf38 +-> ssh-ed25519 aY2AXA kY842plwuQkpLhVF/NSF0fgibmy4fDKT29BI8QPSuFY +52hfgDE3BE5BlMeoXEL7DbFpKx8WiGHFj+g1GCNHo/o +-> ssh-ed25519 Nl/5yA aXblBrmFnTOYlaDt5ULrxBSudS1lkc5Y4GRBDzYEwE0 +sh+dbHuQ/GPViOEu/iwrQVmwMEFZt2U4cd8fqtBhips +-> ssh-ed25519 GdLgCQ qOepkvfW8tb5zx8mx0ZEfHlsTQqbAVuSNzltRtFXyBI +T4dHO/iGupNu7QpJB88AyfWGUiy18lUlBjEMEE1TBNc +-> ssh-ed25519 tOH/HQ 58lz7Qw3cMgSQuSbRfLwvv+RLtqyZCmKiSQuStmVCDg +lZ4PpHldp9Kmh9u+JuUuHp5CtBEQ0s8qcrhJY+qL1ro +-> ssh-ed25519 FpzvfQ pIu/O2EfGHOdIIv9C5RudJe0wE0tF2a2zN+uBFwqpWg +FwU79gBV4WXtPK3XO/EEcMagF5MX66qIZekgdh6VxIg +-> ssh-ed25519 kdPvzQ WgoFNx6cxQ3OaneO3s9wjAJhU8r232rsEmSekU1GXAk +LagJMqiT6t21PfgXaz1c9Vb3Us2mxB+rUByBUhD5SyM +-> XWz-grease ~X,$Oko +qcqccJwfeg +--- Q+UD1YpBDrJ3Ltv283DDrr3AY/MN46qqU/DHb3EcMj0 + +o8缹lOX"aq"E(_fɗ ,t4pNJYX{o^M +Іws 85yُl \ No newline at end of file diff --git a/secrets/gitlab/myself-qemu-runner-reg-3.age b/secrets/gitlab/myself-qemu-runner-reg-3.age new file mode 100644 index 0000000000000000000000000000000000000000..50e0948572fbda04294ad533b3c72da62d583bf9 GIT binary patch literal 1117 zcmZY4yUXJQ003}x5#sqDh{i+2qt-myJbH?fq#5Ibvia_ad@q$VyHua@ta^> zC*vqYF$|f3xt8nO7=*S2L&8WvPiAS*p{3(uQ>Mi0WSU5YE>!_ei#>{D(qsw@m=Y^z zMDu2W5K0qqPar{oAc}~UClz+d6~>>SJmiv4ysE7hsk!OFhB!g4b#Ub@JH&y28BvEi zT}yC~lld*%Dw0L!I6Bg-0$BP!6jXIMqaYcwCuUT0cFjb@+OT>tO5rod6uKay(-gj{ ztyd`JQJ{d;8E@BwtTBY8N6R&X>GWRd60ZuNU6M2IS(A?#xtNnc@9OH(ZA;g7GkNO9 zN)o26*~3%HLI#44L)A(0kmlFw@~WopA?!qfkdBn5sT-L+BzB;BvQBP!f4~!k*{(X& zOO)VD$Vw{AVW4Wmla-2%iDC?W$nB*aECn83REyTxLaaxjb1jSODiL>AH9nMM#sINW zZ%m^Z*ri-&E{rXZ!^{{~Ck}I3OBf{25-CDm*ti=2Qz!330`oJ44QL zGdo1HBvy3JA3UWJf~#7i36Nlby8@5wPR7kXZ%RJGpk7(N}JmW?z z0RtaBb$a2UaQXcYPf{Oz@!YdFZr=Fz`XAptm_7aFQKH)O#c#&n=#SxT{+mnplV2|X zWZ!y5$liZk?E2Sd_wS{5AKkur|62WFcjx&BU%m0q_|8i||HVA<`=_7Xx_sn(&Asd2 dyZGr>`t$Aex8J#Sara^K7=Q8O_v(vl{{pdudcFVv literal 0 HcmV?d00001 diff --git a/secrets/gitlab/myself-qemu-runner-reg-4.age b/secrets/gitlab/myself-qemu-runner-reg-4.age new file mode 100644 index 0000000000000000000000000000000000000000..a7bdccebe0c4e57c6b8bbe9286faa906e166c06a GIT binary patch literal 1241 zcmZY6$;;ye0LEc&O7P}@hzddQyTne{Yy*mEnx;*fW@(xY1hXg2o@Qx++d(hN2QuR5 zK@bE5K~TZLlgm-?B7zQhn8SeHM1?^IQBXX7{sj;3?=^jQ;HGxj{IJ6)5hu&kmwoEQg-8%!z|4@xGL5GB zsHb}oQAYjFv>;w$xL!)iMwGXVun$pt8*mr3Z9IfK0_4S2NvnlRJ7H5Lox(19pP<~+ zv8pz#B)~*Un=#fWj*0;$tA_>_Scp_CkSt(Rjfq{ec85h-WEiy#L#SSb+o~R}FKV>2 z@ktTPPNNJy#mn50D5x@$WO?#*yhUXJ%ApV;^L&97CX}E`3qV2zv15bWT1^(Xzm%Us{PeUQ#WwNQM8EtoYS%C4%!pK2*n@L8b-+WhjirBS#k{q*3-=SMXzD z%}NspMYWg%B!u)rA@`Tbfzv$&g>!-y)R7q1{3L9Of6EMYN(;suvA?)&nfHw+f}9E& zYn^wvid;KW%9@eNE~9&^agrO|Z%j*1-q7X%@hastgrU@)0h^WUH{}7^;g| zk6J>|G-gK7EH0qLMJ;lKrWZjrYRhJ^-bis<* z^#kzi5%5&2;FHMuDJ}zA#Y7aB>jQgMs$811-jZlcRiz`j+(Ej9ik4ReMVGsp6Hs)S z;HuxbA|5EiM3_fYpwE%MW&_wT3;kFIWjC(6cq)Ym-5=&hjp7$w%QDquMYt9do1?MP zLNY`Mql(j2v&-T@Jl3+A z0i$2giu)k9q2I*wvxk1{-+uX?YrFI4%D12Yah3h@ueUA%#?>#rd*S&v`0IDyFW&oW z^FVF9`^mF6o_qNA`*(iuAAISjo5l4zAAY6%dDpAAAHO2Le)hdG+`jg)ORg`ya_z-$ e-u(IN8-G9g&iSodAANS;dpC>o`uD4!%KrdDy{ZcU literal 0 HcmV?d00001 diff --git a/secrets/gitlab/myself-qemu-runner-reg-5.age b/secrets/gitlab/myself-qemu-runner-reg-5.age new file mode 100644 index 0000000..3b9b1e4 --- /dev/null +++ b/secrets/gitlab/myself-qemu-runner-reg-5.age @@ -0,0 +1,21 @@ +age-encryption.org/v1 +-> ssh-ed25519 mOmPfg w03rWuYjifXp2/Kdv39PU6Fx8ttD+BcPMVZkJdfRWEE +iXmxJd9pW1Xc8XtvMF6wOtyjvPzGeDiaKuF28W0+MiM +-> ssh-ed25519 YJiRbw YDx0gI3Gk5YBLImABvTh+Vl8WVURbPjdZbme0zNmkXI +OMty1bEO4Mlr01GSwZ9Ig3DBylsObUcck67B6kE7zEk +-> ssh-ed25519 aY2AXA 57Mvd6nFNFw7Sh8JOm4kuaetSNatG8f6J+ExIsHp03s +pCqA6SmOWvPaP/ydWz24ZeFqP+Z11hm94F1Bey4s+uk +-> ssh-ed25519 Nl/5yA TwMhEFDBL9Y8jSgF6F2SE+QunjaVWGm+LCdEuLEl/j0 +oinaKzMAlTejndkZUjCMnIgS7FLxWOfO/w24TCS3IrA +-> ssh-ed25519 GdLgCQ aMAkHj3LfAobDICK4OMaAeyYFMLt1Evx5WmVl1bBMHg +/cVfO29x1jTdyeSpauhKNzt1C9+T4z1kIujIPORtn/c +-> ssh-ed25519 tOH/HQ daRvGZwMGnFRFKAxHcOR3Pgb2E+h8y7cQK5G+vDXVls +olWk4ahhkH/dNPyUhxI8iWTlNca6hpxVZqkbvJCU9mQ +-> ssh-ed25519 FpzvfQ zcUzSRK67CLrn6KBe2AHIb0hop3smip4y2+2rC3yznU +ea3y+ZhzzT7Kk37g84G6Lm5Hmo+7l8wxuwmnatIM5xo +-> ssh-ed25519 kdPvzQ f7bGPzHX2pxUSxMI78M15deiV3rXN+v3p2E52v6JiS4 +UETYJIs0xo4UakXgqjf+gthl6L2VuTBeQTRyv7/C3k0 +-> I-grease nF) +EyWKPe3DezYJKTUV/fFxjTZ0MEVYnEWmuSH7bGzqqEBLfoI +--- manrceW5TDDoWUUWloP3bIGIkk1jXY4YA8vI82W8P04 +ƝPdٕ޵%1|' ZK'[@yF?'i{ !f> bt~zq$$]HBRL#r+@\ \ No newline at end of file diff --git a/secrets/gitlab/myself-vbox-runner-reg.age b/secrets/gitlab/myself-vbox-runner-reg.age new file mode 100644 index 0000000..40818f0 --- /dev/null +++ b/secrets/gitlab/myself-vbox-runner-reg.age @@ -0,0 +1,22 @@ +age-encryption.org/v1 +-> ssh-ed25519 mOmPfg 1+SrnSmUVltI+CShLbsLlXukVf3L/W4Zewwu6kYS020 +AYynTBanXxis2sadfTbCEfbK071mbYAp52H0GgdswO0 +-> ssh-ed25519 YJiRbw ++Rpjl9AUg1iiKueeoPwceiuAUsmhEzHNKj6zAwrQFo +oKRmh0LZS8XOGsQ55BvIvLFvybWkTuOj8ZN+f4FcqN8 +-> ssh-ed25519 aY2AXA IUkslHupkzLK2oXpcj17LfzsCdJ5RaIt2V13OKQEnzw +VLHEbGXmIPiJa49/oS1FCzpIQWjlG2XJ3SnVRyPcdZg +-> ssh-ed25519 Nl/5yA HWZHuESxYQIX0Ol1Cd0SQGHsAlBSCcvt6zyKtyROIy0 +kNDdxcZ+MA+CDk2fU1wLu0iCL/6528gVIGy3EyrgX7Y +-> ssh-ed25519 GdLgCQ ZQgnEBjjWXo9CKvA8VayduzEqzhu2LL5OzKbF6R8PmE +6BMycP8rCP9xb2tLscpiC7o0ORXju3EVPhge6O8o7QU +-> ssh-ed25519 tOH/HQ Pw0XAGp2pCJEGH8KqjBbPFZqecUaOBJTBWc7Vbbe4T0 +IOsaS11c6MF0uAJAAkeE4vn63mLCaqlOG8/2G2XtAsk +-> ssh-ed25519 FpzvfQ nG48AJt+9ArEPeFSfhbem4H8Oxd7d7oOMNm15oosF20 +IslIFcLR9d4wfGgdOhzHd8O2Vpc+WTW4h1YNnPy1oyI +-> ssh-ed25519 kdPvzQ 7CUogLrCwPqHowodCayIUKVebjrTP5/NNoJXFCEegXo +ep+5sIDKuoxY50DlWUMS2MgKUwoG8PjgHaTeBKmd87o +-> x+-grease 1 Wd/1m/ +Fmb8Vc497/c5m3sWBtcg11fdQsK/YTDXBEg+DToWFvA8C8GMYGk2VIaCcC4IHH3y +6onJXg9p9e6P825Tavxbdgu74ZAcsIZtvkt7 +--- +spumB58VEThrLeLalw56bfovG0jxH+AZhRwyM7O8lQ +HdJDhLx1HDx귟#y!uA'[CxS7]"<\]{2IHew ;67!tUlb)rԁ`eƍ{7az \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 04482a9..25a4be8 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -49,5 +49,10 @@ in # Then pipe the resulting files to agenix -e "gitlab/key.age".publicKeys = everyone; "gitlab/cert.age".publicKeys = everyone; - "gitlab/myself-qemu-runner-reg.age".publicKeys = everyone; + "gitlab/myself-qemu-runner-reg-1.age".publicKeys = everyone; + "gitlab/myself-qemu-runner-reg-2.age".publicKeys = everyone; + "gitlab/myself-qemu-runner-reg-3.age".publicKeys = everyone; + "gitlab/myself-qemu-runner-reg-4.age".publicKeys = everyone; + "gitlab/myself-qemu-runner-reg-5.age".publicKeys = everyone; + "gitlab/myself-vbox-runner-reg.age".publicKeys = everyone; }