Merge branch 'try-new-image' into 'main'

Use new image by default

Closes #1

See merge request greg/vms!10

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -7,6 +7,8 @@ serial-output*
 grub.cfg
 isolinux.cfg
 Vagrantfile
+.direnv
+result
 
 # Generated distros
 distros/fedora/rawhide/*.hcl

.gitlab-ci-build.yml.in

@@ -11,9 +11,6 @@ stages:
           - sources/build.pkr.hcl
       paths:
         - ./packer_config/
-    - key: isofiles
-      paths:
-        - cache/*.iso
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -23,7 +20,7 @@ stages:
         EXCEPT: '-except=upload'
   needs:
     - pipeline: $PARENT_PIPELINE_ID
-      job: distros
+      job: continue
       artifacts: true
     - pipeline: $PARENT_PIPELINE_ID
       job: generate
@@ -42,9 +39,6 @@ stages:
       -var memory=4096
       ${EXCEPT}
       sources
-  before_script:
-    - packer init sources
-    - echo $COMMAND
   script:
     - echo $COMMAND > build.sh
     - bash build.sh || exit 1
@@ -56,7 +50,7 @@ qemu.amd64:
     only: "-only=qemu.amd64"
     tag: qemu
   parallel:
-    matrix: []
+    matrix: QEMU_MATRIX
 
 virtualbox-iso.amd64:
   extends: .build
@@ -65,11 +59,7 @@ virtualbox-iso.amd64:
     only: '-only="virtualbox-iso.amd64"'
     tag: vbox
   parallel:
-    matrix: []
+    matrix: VBOX_MATRIX
-  after_script: |-
-    if [ "$CI_JOB_STATUS" == "success" ]; then
-      xonsh test.xsh
-    fi
 
 hyperv-iso.amd64:
   extends: .build
@@ -78,15 +68,11 @@ hyperv-iso.amd64:
     only: '-only="hyperv-iso.amd64"'
     tag: hyperv
   parallel:
-    matrix: []
+    matrix: HYPERV_MATRIX
   script:
     - echo $env:COMMAND > build.ps1
     - echo 'if (-not $?) { throw "Error in build"}' >> build.ps1
     - pwsh .\build.ps1
-  after_script: |-
-    if ( "$CI_JOB_STATUS" -eq "success" ) {
-      xonsh test.xsh
-    }
 
 "Create release":
   stage: release
@@ -96,7 +82,7 @@ hyperv-iso.amd64:
   script: echo "Tagging as released"
   needs:
     - pipeline: $PARENT_PIPELINE_ID
-      job: distros
+      job: continue
       artifacts: true
   release:
     tag_name: '$CI_COMMIT_TAG'

.gitlab-ci.yml

@@ -1,41 +1,42 @@
 stages:
   - generate
-  - lint
+  - check
-  - cache
   - trigger
 
 variables:
   PACKER_CONFIG_DIR: ./packer_config/
   PACKER_CACHE_DIR: ./cache/
 
+default:
+  image: registry.thehellings.com/greg/ci-images/vm-test:latest
+  tags:
+    - nix
+
 generate:
   stage: generate
-  cache:
-    - key: isofiles
-      paths:
-        - cache/*.iso
   artifacts:
     paths:
       - distros/**/*.pkrvars.hcl
-  script: |-
+  script: nix run ".#generate"
-    for vars in $(find distros/ -name '*.pkrvars.hcl.sh'); do
-      "./${vars}"
-    done
 
-distros:
+continue:
   stage: generate
-  image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
   artifacts:
     reports:
       dotenv: build.env
     paths:
       - .gitlab-ci-build.yml
-  script: |-
+  script:
-    xonsh ./build.xsh -a --list
+    - nix build ".#continue"
-    echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env
+    - cp result .gitlab-ci-build.yml
+    - cat result
+
+shellcheck:
+  stage: generate
+  script: nix run ".#shellcheck"
 
 validate:
-  stage: lint
+  stage: check
   needs:
     - generate
   cache:
@@ -44,45 +45,28 @@ validate:
           - sources/build.pkr.hcl
       paths:
         - ./packer_config/
-  script: |-
+  script: nix run ".#validate"
-    set -exo pipefail
-    packer init sources
-    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
-      packer validate -except=upload -var-file=${vars} sources/
-    done
 
-shellcheck:
+# Makes sure that all of the ISO files we are looking for are
-  stage: lint
+# present on the S3 buckets that we use
-  script: |-
-    shellcheck --version
-    shellcheck $(find . -name '*.sh')
-
-# Downloads all the ISO files so that we have a single cache with all
-# of them present. This prevents a race condition where some of the
-# files were being downloaded multiple times due to parallel jobs, and
-# others were disappearing (e.g. daily builds for Rawhide) before we
-# could get to them
 downloads:
-  stage: cache
+  stage: check
-  image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
+  retry: 2
+  needs:
+    - generate
   cache:
-    - key: isofiles
-      paths:
-        - cache/*.iso
     - key:
         files:
           - sources/build.pkr.hcl
       paths:
         - ./packer_config/
-  script:
+  script: nix run ".#cache"
-    - packer init sources
-    - bash cache.sh
 
 trigger:
   stage: trigger
   trigger:
     include:
       - artifact: .gitlab-ci-build.yml
-        job: distros
+        job: continue
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID

build.xsh

@@ -1,93 +0,0 @@
-#!/usr/bin/env xonsh
-from pathlib import Path
-import sys
-sys.path.insert(0, str(Path(__file__).parent / "lib"))
-from support import Support, BASE
-import time
-import json
-
-from argparse import ArgumentParser
-from functools import reduce
-from yaml import dump, Dumper, load, Loader
-
-source test.xsh
-
-
-year, month, day, hour, minute, second, wday, jday, dst = time.localtime()
-STAMP = f"build={year}.{month}.{day}.{hour}"
-
-support = Support()
-parser = ArgumentParser("Build a box")
-parser.add_argument("--distro", "-d", help="Name of the distro/ folder")
-parser.add_argument("--version", "-v", help="Version of the distro to build")
-parser.add_argument("--build", "-b", help="Name of the target build, e.g. qemu.amd64, virtualbox-iso.amd64")
-parser.add_argument("--list", "-l", help="List all build targets", action="store_true")
-parser.add_argument("--all", "-a", help="Build all applicable targets", action="store_true")
-parser.add_argument("--upload", "-u", help="Force doing uploads", action="store_true")
-parser.add_argument("--headless", help="Run builds headless", action="store_true")
-parser.add_argument("--stamp", "-s", help="Time stamp for builds", default=STAMP)
-args = parser.parse_args()
-
-
-my_list = []
-
-
-# Now provide menus if we don't have defaults
-def process_all():
-    if args.distro:
-        process_distro(support.distros[args.distro])
-    else:
-        for name, distro in support.distros.items():
-            process_distro(distro)
-
-def process_distro(distro):
-    if args.version:
-        process_builds(distro.builds_for_version(args.version))
-    else:
-        process_builds(distro.builds)
-
-def process_builds(builds):
-    for build in builds:
-        if args.build:
-            name, arch = args.build.split(".")
-            if build.provider.name == name and build.provider.arch == arch:
-                do_build(build)
-        else:
-            do_build(build)
-
-def do_build(build):
-    if args.headless:
-        headless = "headless=true"
-    else:
-        headless = "headless=false"
-
-    if args.list:
-        my_list.append(build)
-    else:
-        if args.upload:
-            if "VAGRANT_CLOUD_TOKEN" not in ${...}:
-                print("You need to set your cloud token")
-                sys.exit(1)
-            packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) @(BASE / "sources")
-        else:
-            upload = "-except=upload"
-            # Skip building file if we already have built it
-            output = Path(__file__).parent / "output" / get_vagrant_provider(build.provider.name) / f"{build.distro}-{build.version}-{build.provider.arch}.box"
-            if output.exists():
-                print(f"Skipping {build} as box already exists")
-            else:
-                packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) -except=upload @(BASE / "sources")
-
-if args.all or args.list or (args.distro and args.version and args.build):
-    process_all()
-    if args.list:
-        with open(".gitlab-ci-build.yml.in", "r") as fp:
-            y = load(fp, Loader=Loader)
-        for item in my_list:
-            print(item)
-            if item.is_github:
-                y[ item.only[6:] ]["parallel"]["matrix"].append({"distro": item.var_path})
-        with open(".gitlab-ci-build.yml", "w") as fp:
-            dump(y, fp, Dumper=Dumper)
-else:
-    print("Doing menu stuff")

cache-restore.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET"
+
+for f in $(mc ls --json storage/isos/cache | jq '.key' | tr -d '"'); do
+	if [ ! -e "./cache/$f" ]; then
+		mc get -q "storage/isos/cache/$f" "./cache/$f"
+	else
+		echo "Skipping already present: $f"
+	fi
+done

cache.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-set -eo pipefail
-
-files=()
-
-# shellcheck disable=SC2044
-for vars in $(find distros/ -name '*.pkrvars.hcl'); do
-  file=$(echo var.iso.url | packer console -var-file="${vars}" -config-type=hcl2 sources/)
-  base="$(basename "${file}")"
-  if [ -e "cache/${base}" ]; then
-    echo "Found cache/${base} already"
-  else
-    files+=("-O" "${file}")
-  fi
-done
-
-mkdir -p cache
-pushd cache
-echo "Downloading ${files[*]}"
-curl --parallel -L "${files[@]}"
-popd

del_boxes.xsh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-virsh -q vol-list default | awk '{print $1}' | xargs -r -L 1 virsh vol-delete --pool default
-sudo virsh -q vol-list default | awk '{print $1}' | sudo xargs -r -L 1 virsh vol-delete --pool default
-vagrant box list | grep virtualbox | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider virtualbox --force
-vagrant box list | grep libvirt | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider libvirt --force

distros/almalinux/9/amd64.pkrvars.hcl

@@ -1,13 +1,14 @@
 distro = "almalinux"
 version = "9"
 iso = {
-	url = "https://mirrors.kernel.org/almalinux/9.3/isos/x86_64/AlmaLinux-9.3-x86_64-boot.iso"
+	url = "https://mirrors.kernel.org/almalinux/9.4/isos/x86_64/AlmaLinux-9.4-x86_64-boot.iso"
-	checksum = "af5377a1d16bbe599ea91a8761ad645f2f54687075802bdc0c0703ee610182e9"
+	checksum = "1e5d7da3d84d5d9a5a1177858a5df21b868390bfccf7f0f419b1e59acc293160"
 }
 boot_command = [
   "<up>e<wait>",
   "<down><down><end><wait>",
   " inst.text inst.ks=cdrom:/ks.cfg",
+  " console=ttyS0 inst.notmux",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 cd_files = [ "distros/almalinux/9/disc/*" ]

distros/almalinux/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "8";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "9";
+	arch = "amd64";
+	eol = false;
+} ]

distros/almalinux/supports.yml

@@ -1,10 +0,0 @@
-8: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-9: *supp

distros/alpine/default.nix

@@ -0,0 +1,5 @@
+[ {
+	version = "3.19.0";
+	arch = "amd64";
+	eol = false;
+} ]

distros/alpine/supports.yml

@@ -1,7 +0,0 @@
-"3.19.0":
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/archlinux/default.nix

@@ -0,0 +1,5 @@
+[ {
+	version = "rolling";
+	arch = "amd64";
+	eol = false;
+} ]

distros/archlinux/rolling/amd64.pkrvars.hcl

@@ -5,8 +5,8 @@ iso = {
   checksum = "52aea58f88c9a80afe64f0536da868251ef4878de5a5e0227fcada9f132bd7ab"
 }
 boot_command = [
-  "e<down><down><down><end> net.ifnames=0 biosdevnames=0<f10>",
+  "e<down><down><down><end> net.ifnames=0 biosdevnames=0 console=ttyS0<f10>",
-  "<wait30>", # Wait for system to come up?
+  "<wait60>", # Wait for system to come up?
   "curl -o /tmp/install.sh http://{{ .HTTPIP }}:{{ .HTTPPort }}/install.sh<enter>",
   "<wait5>",
   "/usr/bin/bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}<enter>"

distros/archlinux/supports.yml

@@ -1,9 +0,0 @@
-rolling:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/centos-stream/8/amd64.pkrvars.hcl.sh-eol

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!env bash
 
 set -ex
 

distros/centos-stream/9/amd64.pkrvars.hcl.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!env bash
 
 set -ex
 
@@ -24,6 +24,7 @@ boot_command = [
   "<up>e<wait>",
   "<down><down><end><wait>",
   " inst.text inst.ks=cdrom:/ks.cfg",
+  " inst.notmux console=ttyS0",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 cd_files = [ "distros/centos-stream/9/disc/*" ]

distros/centos-stream/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "8";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "9";
+	arch = "amd64";
+	eol = false;
+} ]

distros/centos-stream/supports.yml

@@ -1,10 +0,0 @@
-8: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-9: *supp

distros/centos/7/amd64.pkrvars.hcl

@@ -8,6 +8,7 @@ boot_command = [
   "<up>e<wait>",
   "<down><down><end><wait>",
   " inst.ks=cdrom:/ks.cfg<wait>",
+  " inst.notmux console=ttyS0",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 cd_files = ["distros/centos/7/disc/*"]

distros/centos/default.nix

@@ -0,0 +1,5 @@
+[ {
+	version = "7";
+	arch = "amd64";
+	eol = false;
+} ]

distros/centos/supports.yml

@@ -1,9 +0,0 @@
-7:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/debian/10/amd64.pkrvars.hcl

@@ -16,6 +16,7 @@ boot_command = [
  "debconf/frontend=noninteractive ",
  "hostname={{.Name}} ",
  "domain=vagrant ",
+ "console=ttyS0 ",
  "<f10>"
 ]
 http_directory = "distros/debian/10/http/"

distros/debian/11/amd64.pkrvars.hcl

@@ -16,6 +16,7 @@ boot_command = [
  "debconf/frontend=noninteractive ",
  "hostname={{.Name}} ",
  "domain=vagrant ",
+ "console=ttyS0 ",
  "<f10>"
 ]
 http_directory = "distros/debian/11/http/"

distros/debian/12/amd64.pkrvars.hcl

@@ -16,6 +16,7 @@ boot_command = [
  "debconf/frontend=noninteractive ",
  "hostname={{.Name}} ",
  "domain=vagrant ",
+ "console=ttyS0 ",
  "<f10>"
 ]
 http_directory = "distros/debian/12/http/"

distros/debian/default.nix

@@ -0,0 +1,20 @@
+[ {
+	version = "9";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "10";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "11";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "12";
+	arch = "amd64";
+	eol = false;
+	exclude = [
+		"virtualbox-iso"
+	];
+} ]

distros/debian/supports.yml

@@ -1,17 +0,0 @@
-10: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-11: *supp
-12:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/default.nix

@@ -0,0 +1,13 @@
+{
+	almalinux = import ./almalinux;
+	alpine = import ./alpine;
+	archlinux = import ./archlinux;
+	centos = import ./centos;
+	centos-stream = import ./centos-stream;
+	debian = import ./debian;
+	fedora = import ./fedora;
+	nixos = import ./nixos;
+	opensuse = import ./opensuse;
+	rockylinux = import ./rockylinux;
+	ubuntu = import ./ubuntu;
+}

distros/fedora/39/amd64-ks.cfg

@@ -1,5 +1,5 @@
 user --name=vagrant --password=vagrant
-rootpw vagrant
+rootpw --plaintext vagrant
 
 text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931
 lang en_US.UTF-8
@@ -7,7 +7,6 @@ keyboard us
 timezone --utc Etc/UTC
 
 selinux --enforcing
-rootpw --lock --iscrypted locked
 
 firewall --disabled
 

distros/fedora/39/amd64.pkrvars.hcl

@@ -6,7 +6,7 @@ iso = {
 }
 boot_command = [
   "<up>e<down><down><end> ",
-  "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0<f10>"
+  "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0<f10>"
 ]
 cd_files = [
   "distros/fedora/39/*.cfg"

distros/fedora/40/amd64-ks.cfg

@@ -0,0 +1,182 @@
+user --name=vagrant --password=vagrant
+rootpw --plaintext vagrant
+
+text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931
+lang en_US.UTF-8
+keyboard us
+timezone --utc Etc/UTC
+
+selinux --enforcing
+
+firewall --disabled
+
+# We pass net.ifnames=0 because we always want to use eth0 here on all the cloud images.
+bootloader --timeout=1 --location=mbr --append="no_timer_check net.ifnames=0 console=tty1 console=ttyS0,115200n8 biosdevnames=0"
+
+services --enabled=sshd
+
+# Configure for gpt with bios+uefi
+clearpart --all --initlabel --disklabel=gpt
+part prepboot  --size=4    --fstype=prepboot
+part biosboot  --size=1    --fstype=biosboot
+part /boot/efi --size=100  --fstype=efi
+part /boot     --size=1000  --fstype=ext4 --label=boot
+part btrfs.007 --size=2000 --fstype=btrfs --grow
+btrfs none --label=fedora btrfs.007
+btrfs /home --subvol --name=home LABEL=fedora
+btrfs /     --subvol --name=root LABEL=fedora
+
+repo --name=fedora --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch
+repo --name=updates --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=updates-released-f$releasever&arch=$basearch
+url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch
+
+reboot
+
+##### begin package list #############################################
+%packages --instLangs=en
+
+# Include packages for the cloud-server-environment group
+@^cloud-server-environment
+
+# Install the tracer dnf plugin to enable automatic reboots
+# IF the user requests package updates
+# AND requests a reboot
+# AND the packages updated require a reboot.
+# https://fedoraproject.org/wiki/Changes/Automatic_Cloud_Reboot_On_Updates
+python3-dnf-plugin-tracer
+
+# Don't include the kernel toplevel package since it pulls in
+# kernel-modules. We're happy for now with kernel-core.
+-kernel
+kernel-core
+
+# Don't include dracut-config-rescue. It will have dracut generate a
+# "rescue" entry in the grub menu, but that also means there is a
+# rescue kernel and initramfs that get created, which (currently) add
+# about another 40MiB to the /boot/ partition. Also the "rescue" mode
+# is generally not useful in the cloud.
+-dracut-config-rescue
+
+# Plymouth provides a graphical boot animation. In the cloud we don't
+# need a graphical boot animation. This also means anaconda won't put
+# rhgb/quiet on kernel command line
+-plymouth
+
+# Install qemu-guest-agent https://pagure.io/cloud-sig/issue/319 To
+# improve the integration with OpenStack and other VM management
+# systems (oVirt, KubeVirt).
+qemu-guest-agent
+
+
+# No need for firewalld for now. We don't have a firewall on by default.
+-firewalld
+
+# Don't include the geolite2 databases, which end up with 66MiB
+# in /usr/share/GeoIP
+-geolite2-country
+-geolite2-city
+
+# Specific to vagrant
+rsync
+fuse-sshfs
+%end
+##### end package list ###############################################
+
+
+##### begin kickstart post ###########################################
+%post --erroronfail
+#!/bin/bash
+set -x
+if [ -e /dev/vda ]; then
+	device="/dev/vda"
+elif [ -e /dev/sda ]; then
+	device="/dev/sda"
+else
+	echo "No device found"
+	exit 1
+fi
+
+virt="$(systemd-detect-virt)"
+if [ "${virt}" == "microsoft" ]; then
+	dnf install -y hyperv-daemons
+	systemctl enable hpervfcopyd
+	systemctl enable hypervkvpd
+	systemctl enable hypervvssd
+fi
+
+if [ "$(arch)" = "x86_64" ]; then
+# Set up legacy BIOS boot if we booted from UEFI
+grub2-install --target=i386-pc "${device}"
+fi
+
+# Blivet sets pmbr_boot flag erroneously and we need to purge it
+# otherwise it'll fail to boot
+parted "${device}" disk_set pmbr_boot off
+
+# linux-firmware is installed by default and is quite large. As of mid 2020:
+#   Total download size: 97 M
+#   Installed size: 268 M
+# So far we've been fine shipping without it so let's continue.
+# More discussion about this in #1234504.
+echo "Removing linux-firmware package."
+rpm -e linux-firmware
+
+# See the systemd-random-seed.service man page that says:
+#   " It is recommended to remove the random seed from OS images intended
+#     for replication on multiple systems"
+echo "Removing random-seed so it's not the same in every image."
+rm -f /var/lib/systemd/random-seed
+
+echo "Import RPM GPG key"
+releasever=$(rpm --eval '%{fedora}')
+basearch=$(uname -i)
+rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+
+echo "Zeroing out empty space."
+# Create zeros file with nodatacow and no compression
+touch /var/tmp/zeros
+chattr +C /var/tmp/zeros
+# This forces the filesystem to reclaim space from deleted files
+dd bs=1M if=/dev/zero of=/var/tmp/zeros || :
+echo "(Don't worry -- that out-of-space error was expected.)"
+# Force sync to disk (Cf. https://pagure.io/cloud-sig/issue/340#comment-743430)
+btrfs filesystem sync /
+rm -f /var/tmp/zeros
+btrfs filesystem sync /
+
+# When we build the image a networking config file gets left behind.
+# Let's clean it up.
+echo "Cleanup leftover networking configuration"
+rm -f /etc/NetworkManager/system-connections/*.nmconnection
+#*/
+
+# Truncate the /etc/resolv.conf left over from NetworkManager during the
+# kickstart. This causes delays in boot with cloud-init because the
+# 192.168.122.1 DNS server cannot be reached.
+truncate -s 0 /etc/resolv.conf
+
+# Clear machine-id on pre generated images
+truncate -s 0 /etc/machine-id
+
+# Vagrant setup
+sed -i 's,Defaults\\s*requiretty,Defaults !requiretty,' /etc/sudoers
+echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant-nopasswd
+sed -i 's/.*UseDNS.*/UseDNS no/' /etc/ssh/sshd_config
+
+cat > /etc/ssh/sshd_config.d/10-vagrant-insecure-rsa-key.conf <<EOF
+# For now the vagrant insecure key is an rsa key
+# htfedoratps://github.com/hashicorp/vagrant/issues/11783
+PubkeyAcceptedKeyTypes=+ssh-rsa
+EOF
+
+ssh-keygen -A
+
+mkdir -m 0700 -p /home/vagrant/.ssh/
+curl -L -o /home/vagrant/.ssh/authorized_keys "https://raw.githubusercontent.com/hashicorp/vagrant/main/keys/vagrant.pub"
+chown -R vagrant:vagrant /home/vagrant/.ssh
+chmod 600 /home/vagrant/.ssh/authorized_keys
+chcon -R unconfined_u:object_r:user_home_t:s0 /home/vagrant/.ssh
+
+%end
+##### end kickstart post ############################################
+

distros/fedora/40/amd64.pkrvars.hcl

@@ -0,0 +1,14 @@
+distro  = "fedora"
+version = "40"
+iso = {
+  url      = "https://download.fedoraproject.org/pub/fedora/linux/releases/40/Server/x86_64/iso/Fedora-Server-dvd-x86_64-40-1.14.iso"
+  checksum = "32d9ab1798fc8106a0b06e873bdcd83a3efea8412c9401dfe4097347ed0cfc65"
+}
+boot_command = [
+  "<up>e<down><down><end> ",
+  "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0<f10>"
+]
+cd_files = [
+  "distros/fedora/40/*.cfg"
+]
+vbox_os_type = "Fedora_64"

distros/fedora/40/finalize.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -ex
+
+dnf -y update
+
+dnf clean all

distros/fedora/40/qemu.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -ex
+
+dnf install -y qemu-guest-agent
+systemctl start qemu-guest-agent
+systemctl enable qemu-guest-agent

distros/fedora/40/virtualbox.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -ex
+
+dnf install -y virtualbox-guest-additions
+systemctl start vboxservice
+systemctl enable vboxservice

distros/fedora/40/vmware.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -ex
+
+dnf install -y open-vm-tools

distros/fedora/default.nix

@@ -0,0 +1,21 @@
+[ {
+	version = "37";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "38";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "39";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "40";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "rawhide";
+	arch = "amd64";
+	eol = false;
+} ]

distros/fedora/rawhide/amd64-ks.cfg

@@ -1,5 +1,5 @@
 user --name=vagrant --password=vagrant
-rootpw vagrant
+rootpw --plaintext vagrant
 
 text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931
 lang en_US.UTF-8
@@ -7,7 +7,6 @@ keyboard us
 timezone --utc Etc/UTC
 
 selinux --enforcing
-rootpw --lock --iscrypted locked
 
 firewall --disabled
 
@@ -77,7 +76,6 @@ qemu-guest-agent
 -geolite2-city
 
 # Specific to vagrant
-dnf-yum
 rsync
 fuse-sshfs
 %end

distros/fedora/rawhide/amd64.pkrvars.hcl.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!env bash
 set -exo pipefail
 
 # Name of the file to write out
@@ -35,7 +35,7 @@ iso = {
 }
 boot_command = [
   "<up>e<down><down><end><bs><bs><bs><bs><bs>",
-  "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0 console=ttyS0<f10>"
+  "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0<f10>"
 ]
 cd_files = [
 	"distros/fedora/rawhide/*.cfg"

distros/fedora/rawhide/qemu.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 set -ex
 
-dnf install -y qemu-guest-agent
 systemctl start qemu-guest-agent
 systemctl enable qemu-guest-agent

distros/fedora/supports.yml

@@ -1,20 +0,0 @@
-37:  # EOL
-38: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-39: *supp
-rawhide:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: hyperv-iso
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso

distros/nixos/23.05/amd64.pkrvars.hcl.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!env bash
 set -exo pipefail
 
 # Name of the file to write out
@@ -34,7 +34,7 @@ boot_command = [
   "sudo bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}<enter>"
 ]
 http_directory = "distros/nixos/23.05/http/"
-boot_wait = "60s"
+boot_wait = "90s"
 EOF
 
 rm -r "${tmp}"

distros/nixos/23.11/amd64.pkrvars.hcl.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!env bash
 set -exo pipefail
 
 # Name of the file to write out
@@ -34,7 +34,7 @@ boot_command = [
   "sudo bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}<enter>"
 ]
 http_directory = "distros/nixos/23.11/http/"
-boot_wait = "60s"
+boot_wait = "90s"
 EOF
 
 rm -r "${tmp}"

distros/nixos/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "23.05";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "23.11";
+	arch = "amd64";
+	eol = false;
+} ]

distros/nixos/supports.yml

@@ -1,10 +0,0 @@
-23.05:  # EOL
-23.11: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/opensuse/15.5/amd64.pkrvars.hcl

@@ -9,6 +9,7 @@ boot_command = [
   "<down><down><down><down><end><wait>",
   "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
   " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1<wait>",
+  " console=ttyS0",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 http_directory = "distros/opensuse/15.5/http/"

distros/opensuse/15.6/amd64.pkrvars.hcl

@@ -9,6 +9,7 @@ boot_command = [
   "<down><down><down><down><end><wait>",
   "<bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs><bs>",
   " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1<wait>",
+  " console=ttyS0",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 http_directory = "distros/opensuse/15.6/http/"

distros/opensuse/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "15.5";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "tumbleweed";
+	arch = "amd64";
+	eol = false;
+} ]

distros/opensuse/supports.yml

@@ -1,11 +0,0 @@
-15.5: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-15.6:
-tumbleweed: *supp

distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh

@@ -1,8 +1,8 @@
-#!/usr/bin/env bash
+#!env bash
 
 name="${0%.sh}"
 
-base="https://download.opensuse.org/tumbleweed/iso/openSUSE-Tumbleweed-NET-x86_64-Current.iso"
+base="https://download.opensuse.org/tumbleweed/iso/openSUSE-Tumbleweed-DVD-x86_64-Current.iso"
 
 # Working directory
 tmp="$(mktemp -d)"
@@ -26,6 +26,7 @@ boot_command = [
   "e<wait>",
   "<down><down><down><down><end><wait>",
   " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1<wait>",
+  " console=ttyS0",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 http_directory = "distros/opensuse/tumbleweed/http/"

distros/opensuse/tumbleweed/http/autoinst.xml

@@ -8,6 +8,7 @@
 			<confirm config:type="boolean">false</confirm>
 			<final_reboot config:type="boolean">true</final_reboot>
 		</mode>
+		<self_update config:type="boolean">true</self_update>
 	</general>
 	<groups config:type="list">
 		<group>

distros/rockylinux/9/amd64.pkrvars.hcl

@@ -1,13 +1,14 @@
 distro = "rockylinux"
 version = "9"
 iso = {
-	url = "http://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9.3-x86_64-boot.iso"
+	url = "https://download.rockylinux.org/pub/rocky/9.4/isos/x86_64/Rocky-9.4-x86_64-boot.iso"
-	checksum = "eb096f0518e310f722d5ebd4c69f0322df4fc152c6189f93c5c797dc25f3d2e1"
+	checksum = "c7e95e3dba88a1f68fff8b7d4e66adf6f76ac4fba2e246a83c46ab79574c78a8"
 }
 boot_command = [
   "<up>e<wait>",
   "<down><down><end><wait>",
   " inst.text inst.ks=cdrom:/ks.cfg",
+  " inst.notmux console=ttyS0",
   "<leftCtrlOn>x<leftCtrlOff>"
 ]
 cd_files = [ "distros/rockylinux/9/disc/*" ]

distros/rockylinux/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "8";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "9";
+	arch = "amd64";
+	eol = false;
+} ]

distros/rockylinux/supports.yml

@@ -1,10 +0,0 @@
-8: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-9: *supp

distros/ubuntu/20.04/amd64.pkrvars.hcl

@@ -7,7 +7,7 @@ iso = {
 boot_wait = "3s"
 boot_command = [
   "e<wait><down><down><down><end><bs><bs><bs>",
-  "console=tty0 ",
+  "console=ttyS0 ",
   "fsck.mode=skip ",
   "locale=en_US ",
   "auto=true ",

distros/ubuntu/22.04/amd64.pkrvars.hcl

@@ -6,7 +6,7 @@ iso = {
 }
 boot_command = [
   "e<down><down><down><end><bs><bs><bs>",
-  "console=tty0 ",
+  "console=ttyS0 ",
   "fsck.mode=skip ",
   "locale=en_US ",
   "auto=true ",

distros/ubuntu/23.10/amd64.pkrvars.hcl

@@ -6,7 +6,7 @@ iso = {
 }
 boot_command = [
   "e<down><down><down><end><bs><bs><bs>",
-  "console=tty0 ",
+  "console=ttyS0 ",
   "fsck.mode=skip ",
   "locale=en_US ",
   "auto=true ",

distros/ubuntu/default.nix

@@ -0,0 +1,17 @@
+[ {
+	version = "18.04";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "20.04";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "22.04";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "23.10";
+	arch = "amd64";
+	eol = false;
+} ]

distros/ubuntu/supports.yml

@@ -1,13 +0,0 @@
-20.04: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-  # linux-azure package broken in 20.04
-22.04: *supp
-"23.10": *supp
-#18.04: *supp  # EOL

flake.lock

@@ -0,0 +1,194 @@
+{
+  "nodes": {
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1717408969,
+        "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "1ebbe68d57457c8cae98145410b164b5477761f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1717285511,
+        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-root": {
+      "locked": {
+        "lastModified": 1713493429,
+        "narHash": "sha256-ztz8JQkI08tjKnsTpfLqzWoKFQF4JGu2LRz8bkdnYUk=",
+        "owner": "srid",
+        "repo": "flake-root",
+        "rev": "bc748b93b86ee76e2032eecda33440ceb2532fcd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "srid",
+        "repo": "flake-root",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1704161960,
+        "narHash": "sha256-QGua89Pmq+FBAro8NriTuoO/wNaUtugt29/qqA8zeeM=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "63143ac2c9186be6d9da6035fa22620018c85932",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1717284937,
+        "narHash": "sha256-lIbdfCsf8LMFloheeE6N31+BMIeixqyQWbSr2vk79EQ=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1718870667,
+        "narHash": "sha256-jab3Kpc8O1z3qxwVsCMHL4+18n5Wy/HHKyu1fcsF7gs=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9b10b8f00cb5494795e5f51b39210fed4d2b0748",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1708475490,
+        "narHash": "sha256-g1v0TsWBQPX97ziznfJdWhgMyMGtoBFs102xSYO4syU=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "0e74ca98a74bc7270d28838369593635a5db3260",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "process-compose": {
+      "locked": {
+        "lastModified": 1718031437,
+        "narHash": "sha256-+RrlkAVZx0QhyeHAGFJnjST+/7Dc3zsDU3zAKXoDXaI=",
+        "owner": "Platonic-Systems",
+        "repo": "process-compose-flake",
+        "rev": "9344fac44edced4c686721686a6ad904d067c546",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Platonic-Systems",
+        "repo": "process-compose-flake",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devshell": "devshell",
+        "flake-parts": "flake-parts",
+        "flake-root": "flake-root",
+        "nixpkgs": "nixpkgs_2",
+        "process-compose": "process-compose",
+        "treefmt-nix": "treefmt-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_3"
+      },
+      "locked": {
+        "lastModified": 1718522839,
+        "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,193 @@
+# github.com/brianmcgee has lots of great ideas for going above and beyond for this
+{
+	description = "Greg's homepage";
+	inputs = {
+		nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+		devshell.url = "github:numtide/devshell";
+		flake-parts.url = "github:hercules-ci/flake-parts";
+		flake-root.url = "github:srid/flake-root";
+		process-compose.url = "github:Platonic-Systems/process-compose-flake";
+		treefmt-nix.url = "github:numtide/treefmt-nix";
+	};
+
+	outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, nixpkgs, ... }@inputs: let
+		lib = nixpkgs.lib;
+		providers = [
+			"qemu"
+			"virtualbox-iso"
+			"hyperv-iso"
+		];
+		distros = import ./distros;
+		# Inserts the distro name into each element in the list
+		insertNames = lib.attrsets.mapAttrsToList (distro: val: (builtins.map (x: x // { inherit distro; }) val)) distros;
+		# Flattens all the distros into a single list
+		flat = lib.lists.flatten insertNames;
+		# Combines providers list with flattened list
+		withProviders = lib.attrsets.mapCartesianProduct
+			({d, provider}: d // { inherit provider; })
+			{ d = flat; provider = providers; };
+		active = builtins.filter (x: (x?eol) && (!x.eol)) withProviders;
+		name = d: "-${d.distro}-${d.arch}-${d.provider}-${d.version}";
+		distroFile = d: "distros/${d.distro}/${d.version}/${d.arch}.pkrvars.hcl";
+		cmd = d: ''
+			version="''${1:-1}"
+			packer build -var "build=$version" -only=${d.provider}.${d.arch} -var-file=${distroFile d}'';
+	in flake-parts.lib.mkFlake { inherit inputs; }
+	{
+		imports = [
+			inputs.devshell.flakeModule
+			inputs.flake-root.flakeModule
+			inputs.treefmt-nix.flakeModule
+		];
+		flake = {
+			# Universal things
+		};
+		systems = [
+			"x86_64-linux"
+			"aarch64-linux"
+		];
+		perSystem = { config, pkgs, self', system, ... }:
+		{
+			_module.args.pkgs = import inputs.nixpkgs {
+				inherit system;
+				config.allowUnfree = true;
+			};
+			# https://numtide.github.io/devshell/intro.html
+			devshells.default = {
+				# These commands will be shown in the motd that users read
+				# upon entering the shell
+				commands = [ {
+					package = self'.packages.cache;
+					category = "utils";
+					help = "Downloads ISO files and adds them to the S3 bucket";
+				} {
+					package = self'.packages.generate;
+					category = "utils";
+					help = "Generate dynamic HCL files";
+				} {
+					package = self'.packages.shellcheck;
+					category = "utils";
+					help = "Run shellchecks";
+				} {
+					package = self'.packages.validate;
+					category = "utils";
+					help = "Validate all the files";
+				} {
+					package = pkgs.vagrant;
+					category = "utils";
+					help = "Vagrant to help with testing";
+				} ] ++ (builtins.map (d: { package = self'.packages."build${name d}"; category = "builds"; }) withProviders);
+				env = [{
+					name = "EFI_DIR";
+					value = "${pkgs.OVMF.fd}/FV/";
+				}];
+			};
+
+# This is not special, this is just flakes
+			packages = let
+				base = {
+					runtimeInputs = with pkgs; [
+						packer
+							qemu_full
+							vagrant
+					];
+				};
+				build = d: let
+					vp = x: if x.provider == "qemu" then "libvirt"
+						else if x.provider == "virtualbox-iso" then "virtualbox"
+						else if x.provider == "hyperv-iso" then "hyperv"
+							else "unknown";
+				in pkgs.writeShellApplication (base // {
+					name = "build${name d}";
+					text = ''
+					set -x
+					${cmd d} -except=upload -var headless=false sources/
+					vagrant box add --provider ${vp d} ./output/${vp d}/${d.distro}-${d.version}.box --name test_box --force
+					'';
+					});
+				upload = d: pkgs.writeShellApplication (base // {
+					name = "upload${name d}";
+					text = ''
+					set -x
+					${cmd d} sources/
+					'';
+					});
+				builds = lib.lists.foldr (d: acc: acc // { "build${name d}" = (build d); }) {} withProviders;
+				uploads = lib.lists.foldr (d: acc: acc // { "upload${name d}" = (upload d); }) {} withProviders;
+				qemu_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "qemu") active));
+				vbox_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "virtualbox-iso") active));
+				hyperv_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "hyperv-iso") active));
+			in builds // uploads // {
+				continue = pkgs.writeText "continue.yml"
+					(builtins.replaceStrings ["QEMU_MATRIX" "VBOX_MATRIX" "HYPERV_MATRIX"] [qemu_list vbox_list hyperv_list]
+						(builtins.readFile ./.gitlab-ci-build.yml.in)
+					);
+				cache = pkgs.callPackage (import ./nix/cache.nix) { inherit active distroFile; };
+				cache-restore = pkgs.writeShellApplication {
+					name = "cache-restore";
+					runtimeInputs = with pkgs; [
+						jq
+						minio-client
+						packer
+					];
+					text = builtins.readFile ./cache-restore.sh;
+				};
+				generate = pkgs.writeShellApplication {
+					name = "generate";
+					runtimeInputs = with pkgs; [
+						bash
+						curl
+						jq
+						pup
+					];
+					text = ''
+						shopt -s globstar
+						for vars in distros/**/*.pkrvars.hcl.sh; do
+							bash "./''${vars}"
+						done
+					'';
+				};
+				shellcheck = pkgs.writeShellApplication {
+					name = "shellcheck";
+					runtimeInputs = with pkgs; [
+						shellcheck
+					];
+					text = ''
+						scripts=()
+						shopt -s globstar
+						for f in **/*.sh; do
+							scripts+=("$f")
+						done
+						shellcheck -e SC2239 "''${scripts[@]}"
+					'';
+				};
+				validate = pkgs.writeShellApplication {
+					name = "validate";
+					runtimeInputs = with pkgs; [
+						packer
+					];
+					text = ''
+						shopt -s globstar
+						packer init sources
+						for vars in distros/**/*.pkrvars.hcl; do
+							packer validate -except=upload "-var-file=''${vars}" sources/
+						done
+					'';
+				};
+			};
+
+			# https://numtide.github.io/treefmt/
+			treefmt.config = {
+				inherit (config.flake-root) projectRootFile;
+				package = pkgs.treefmt;
+
+				programs = {
+					deadnix.enable = true;
+				};
+
+				settings.formatter = {
+				};
+			};
+		};
+	};
+}

lib/build.py

@@ -1,40 +0,0 @@
-from provider import Provider
-
-
-class Build:
-    def __init__(self, distro, version, provider: Provider):
-        self.distro: str = distro
-        self.version: str = str(version)
-        self.provider: Provider = provider
-
-    @property
-    def var_file(self):
-        return f"-var-file={self.var_path}"
-
-    @property
-    def matrix(self):
-        return {
-            "builder": str(self.provider),
-            "distro": self.var_path,
-            "runner": self.provider.runner,
-        }
-
-    @property
-    def is_github(self) -> bool:
-        if self.provider.runner is not None:
-            return True
-        return False
-
-    @property
-    def var_path(self):
-        return f"distros/{self.distro}/{self.version}/{self.provider.arch}.pkrvars.hcl"
-
-    @property
-    def only(self):
-        return f"-only={self.provider}"
-
-    def __str__(self) -> str:
-        return f"{self.distro} {self.version}: {self.provider}"
-
-    def __repr__(self) -> str:
-        return str(self)

lib/distro.py

@@ -1,28 +0,0 @@
-from provider import Provider
-from build import Build
-
-
-class Distro:
-    def __init__(self, name: str, support: map):
-        self.name = name
-        self.builds = []
-        for version, matrix in support.items():
-            if not matrix:
-                continue
-            for support in matrix:
-                provider = Provider(support["provider"], support["arch"])
-                build = Build(self.name, version, provider)
-                self.builds.append(build)
-
-    def __str__(self) -> str:
-        return f"{self.name}"
-
-    def __repr__(self) -> str:
-        return f"{self.name}: {' '.join(self.versions)}"
-
-    @property
-    def versions(self) -> set[str]:
-        return set([b.version for b in self.builds])
-
-    def builds_for_version(self, version: str) -> set[Build]:
-        return set(filter(lambda x: x.version == version, self.builds))

lib/provider.py

@@ -1,17 +0,0 @@
-class Provider:
-    def __init__(self, name, arch):
-        self.name = name
-        self.arch = arch
-
-    def __str__(self) -> str:
-        return f"{self.name}.{self.arch}"
-
-    @property
-    def runner(self) -> str:
-        if self.name == "qemu":
-            return "qemu"
-        elif self.name.startswith("virtualbox"):
-            return "virtualbox"
-        elif self.name.startswith("hyperv"):
-            return "hyperv"
-        return None

lib/support.py

@@ -1,23 +0,0 @@
-from distro import Distro
-from provider import Provider
-from pathlib import Path
-from yaml import load, Loader
-
-
-BASE = Path(__file__).parent.parent.resolve()
-
-
-class Support:
-    def __init__(self):
-        self.base = BASE
-        self.distros = self.__get_distros()
-
-    def __get_distros(self) -> map:
-        distros = {}
-        distros_dir = self.base / "distros"
-        for f in distros_dir.glob("**/supports.yml"):
-            name = f.parent.name
-            with open(f, "r") as fp:
-                support_yml = load(fp, Loader=Loader)
-            distros[name] = Distro(name, support_yml)
-        return distros

nix/cache.nix

@@ -0,0 +1,43 @@
+{
+	writeShellApplication,
+	active,
+	distroFile,
+
+	curl,
+	minio-client,
+	packer,
+
+	...
+}:
+
+let
+	inherit (builtins) concatStringsSep map;
+in writeShellApplication {
+	name = "cache-isos";
+	runtimeInputs = [
+		curl
+		minio-client
+		packer
+	];
+	text = ''
+		push="''${1:-yes}"
+		if [ "$push" == "yes" ]; then
+			mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET"
+		fi
+		mkdir -p cache
+	'' + (concatStringsSep "\n" (
+		map (d: ''
+			url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
+			sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
+			# shellcheck disable=SC2143
+			if [ -z "$(mc ls storage/isos/cache/ | grep "$sha")" ]; then
+				curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url"
+				if [ "$push" == "yes" ]; then
+					mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso"
+				fi
+			else
+				echo "Skipping ${d.distro} - cache/$sha.iso"
+			fi
+		'') active
+	));
+}

sources/hyperv-iso.pkr.hcl

@@ -1,6 +1,6 @@
 source "hyperv-iso" "amd64" {
-  iso_urls         = [ "cache/${basename(var.iso.url)}", var.iso.url]
+  iso_urls         = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ]
-  iso_target_path  = "cache/${basename(var.iso.url)}"
+  iso_target_path  = "cache/${var.iso.checksum}.iso"
   iso_checksum     = var.iso.checksum
   output_directory = "output/hyperv/${local.name}"
 

sources/inputs.pkr.hcl

@@ -109,6 +109,16 @@ variable "vbox_os_type" {
   default = "Linux_64"
 }
 
+variable "storage_url" {
+  type    = string
+  default = env("STORAGE_URL")
+}
+
+variable "minio_secret" {
+  type    = string
+  default = env("MINIO_SECRET")
+}
+
 locals {
   name = "${var.distro}-${var.version}"
   description = templatefile("../README.box.md", {

sources/qemu.pkr.hcl

@@ -4,8 +4,8 @@ variable "qemu_accelerator" {
 }
 
 source "qemu" "amd64" {
-  iso_urls         = [ "cache/${basename(var.iso.url)}", var.iso.url]
+  iso_urls         = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ]
-  iso_target_path  = "cache/${basename(var.iso.url)}"
+  iso_target_path  = "cache/${var.iso.checksum}.iso"
   iso_checksum     = var.iso.checksum
   output_directory = "output/libvirt/${local.name}"
 
@@ -35,6 +35,7 @@ source "qemu" "amd64" {
     ["-chardev", "socket,path=${local.name}.qga.sock,server=on,wait=off,id=qga0"],
     ["-device", "virtio-serial"],
     ["-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"],
+    ["-serial", "stdio"]
   ]
   accelerator = var.qemu_accelerator
 

sources/virtualbox-iso.pkr.hcl

@@ -1,6 +1,6 @@
 source "virtualbox-iso" "amd64" {
-  iso_urls         = [ "cache/${basename(var.iso.url)}", var.iso.url]
+  iso_urls         = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ]
-  iso_target_path  = "cache/${basename(var.iso.url)}"
+  iso_target_path  = "cache/${var.iso.checksum}.iso"
   iso_checksum     = var.iso.checksum
   output_directory = "output/virtualbox/${local.name}"
 

test.xsh

@@ -1,88 +0,0 @@
-#!env xonsh
-# vim: set ft=xonsh:
-import argparse
-import pathlib
-import re
-import sys
-import time
-from shutil import rmtree
-
-
-def get_vagrant_provider(provider):
-    "Vagrant provider is not always the same as Packer provider"
-    if provider.startswith("qemu"):
-        return "libvirt"
-    elif provider.startswith("virtualbox"):
-        return "virtualbox"
-    elif provider.startswith("vmware"):
-        return "vmware"
-    elif provider.startswith("hyperv"):
-        return "hyperv"
-    else:
-        return provider
-
-
-def get_providers():
-    "List of providers as defined in sources/"
-    providers: set[str] = {
-        "qemu.amd64"
-        "virtualbox-iso.amd64",
-        "vmware-iso.amd64"
-    }
-    return providers
-
-
-def get_parser(args, unsupported=set()):
-    parser = argparse.ArgumentParser("Builder")
-    parser.add_argument("--headless", action="store_true")
-    parser.add_argument("--upload", action="store_true")
-    parser.add_argument("--provider", action="append")
-    parsed = parser.parse_args(args)
-    opts = {}
-    if parsed.headless:
-        opts["headless"] = "headless=true"
-    else:
-        opts["headless"] = "headless=false"
-    if parsed.upload:
-        opts["upload"] = ""
-    else:
-        opts["upload"] = "-except=upload"
-    if parsed.provider:
-        opts["providers"] = set(parsed.provider)
-    else:
-        opts["providers"] = get_providers()
-    # Filter providers
-    opts["providers"] -= unsupported
-    return opts
-
-
-def main():
-    # Equivalent of `set -e` in Bash
-    $RAISE_SUBPROC_ERROR = True
-    if len(sys.argv) == 2:
-        dry_run = True
-    else:
-        dry_run = False
-    for box in g`output/**/*.box`:
-        p = pathlib.Path(box)
-        with open("Vagrantfile.in", "r") as fr:
-            content = fr.read()
-        content = content.replace("@@BOX@@", p.resolve().as_uri())
-        content = content.replace("@@BOX_NAME@@", p.name)
-        with open("Vagrantfile", "w") as fw:
-            fw.write(content)
-        if not dry_run:
-            if str(p.parent.name) in ["virtualbox", "libvirt", "vmware", "hyperv"]:
-                vagrant --provider @(p.parent.name) box add @(box) --name @(p.name) --force
-                vagrant up --provider @(p.parent.name)
-            else:
-                print(f"Now whatcha wanna go and do that for? {box}")
-                continue
-            vagrant destroy -f
-            vag_folder = pathlib.Path(".vagrant")
-            if vag_folder.exists():
-                rmtree(vag_folder)
-
-
-if __name__ == '__main__':
-    main()