Move almost everything over to Nix

Eliminate Xonsh usage
Eliminate usage of the supports.yml files for default.nix
Point ISOs directly to MinIO URLs instead

.gitignore

@@ -8,6 +8,7 @@ grub.cfg
 isolinux.cfg
 Vagrantfile
 .direnv
+result
 
 # Generated distros
 distros/fedora/rawhide/*.hcl

.gitlab-ci-build.yml.in

@@ -39,10 +39,7 @@ stages:
       -var memory=4096
       ${EXCEPT}
       sources
-  before_script:
-    - nix run ".#cache-restore"
   script:
-    - ls -l cache/
     - echo $COMMAND > build.sh
     - bash build.sh || exit 1
 
@@ -53,7 +50,7 @@ qemu.amd64:
     only: "-only=qemu.amd64"
     tag: qemu
   parallel:
-    matrix: []
+    matrix: QEMU_MATRIX
 
 virtualbox-iso.amd64:
   extends: .build
@@ -62,7 +59,7 @@ virtualbox-iso.amd64:
     only: '-only="virtualbox-iso.amd64"'
     tag: vbox
   parallel:
-    matrix: []
+    matrix: VBOX_MATRIX
 
 hyperv-iso.amd64:
   extends: .build
@@ -71,7 +68,7 @@ hyperv-iso.amd64:
     only: '-only="hyperv-iso.amd64"'
     tag: hyperv
   parallel:
-    matrix: []
+    matrix: HYPERV_MATRIX
   script:
     - echo $env:COMMAND > build.ps1
     - echo 'if (-not $?) { throw "Error in build"}' >> build.ps1

build.xsh

@@ -1,93 +0,0 @@
-#!/usr/bin/env xonsh
-from pathlib import Path
-import sys
-sys.path.insert(0, str(Path(__file__).parent / "lib"))
-from support import Support, BASE
-import time
-import json
-
-from argparse import ArgumentParser
-from functools import reduce
-from yaml import dump, Dumper, load, Loader
-
-source test.xsh
-
-
-year, month, day, hour, minute, second, wday, jday, dst = time.localtime()
-STAMP = f"build={year}.{month}.{day}.{hour}"
-
-support = Support()
-parser = ArgumentParser("Build a box")
-parser.add_argument("--distro", "-d", help="Name of the distro/ folder")
-parser.add_argument("--version", "-v", help="Version of the distro to build")
-parser.add_argument("--build", "-b", help="Name of the target build, e.g. qemu.amd64, virtualbox-iso.amd64")
-parser.add_argument("--list", "-l", help="List all build targets", action="store_true")
-parser.add_argument("--all", "-a", help="Build all applicable targets", action="store_true")
-parser.add_argument("--upload", "-u", help="Force doing uploads", action="store_true")
-parser.add_argument("--headless", help="Run builds headless", action="store_true")
-parser.add_argument("--stamp", "-s", help="Time stamp for builds", default=STAMP)
-args = parser.parse_args()
-
-
-my_list = []
-
-
-# Now provide menus if we don't have defaults
-def process_all():
-    if args.distro:
-        process_distro(support.distros[args.distro])
-    else:
-        for name, distro in support.distros.items():
-            process_distro(distro)
-
-def process_distro(distro):
-    if args.version:
-        process_builds(distro.builds_for_version(args.version))
-    else:
-        process_builds(distro.builds)
-
-def process_builds(builds):
-    for build in builds:
-        if args.build:
-            name, arch = args.build.split(".")
-            if build.provider.name == name and build.provider.arch == arch:
-                do_build(build)
-        else:
-            do_build(build)
-
-def do_build(build):
-    if args.headless:
-        headless = "headless=true"
-    else:
-        headless = "headless=false"
-
-    if args.list:
-        my_list.append(build)
-    else:
-        if args.upload:
-            if "VAGRANT_CLOUD_TOKEN" not in ${...}:
-                print("You need to set your cloud token")
-                sys.exit(1)
-            packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) @(BASE / "sources")
-        else:
-            upload = "-except=upload"
-            # Skip building file if we already have built it
-            output = Path(__file__).parent / "output" / get_vagrant_provider(build.provider.name) / f"{build.distro}-{build.version}-{build.provider.arch}.box"
-            if output.exists():
-                print(f"Skipping {build} as box already exists")
-            else:
-                packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) -except=upload @(BASE / "sources")
-
-if args.all or args.list or (args.distro and args.version and args.build):
-    process_all()
-    if args.list:
-        with open(".gitlab-ci-build.yml.in", "r") as fp:
-            y = load(fp, Loader=Loader)
-        for item in my_list:
-            print(item)
-            if item.is_github:
-                y[ item.only[6:] ]["parallel"]["matrix"].append({"distro": item.var_path})
-        with open(".gitlab-ci-build.yml", "w") as fp:
-            dump(y, fp, Dumper=Dumper)
-else:
-    print("Doing menu stuff")

cache.sh

@@ -4,7 +4,7 @@ set -eo pipefail
 files=()
 
 # These files are always out of date
-rm -f cache/*latest* cache/*Rawhide*
+rm -f cache/*latest* cache/*Rawhide* cache/openSUSE*Current*
 
 # shellcheck disable=SC2044
 for vars in $(find distros/ -name '*.pkrvars.hcl'); do

del_boxes.xsh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-virsh -q vol-list default | awk '{print $1}' | xargs -r -L 1 virsh vol-delete --pool default
-sudo virsh -q vol-list default | awk '{print $1}' | sudo xargs -r -L 1 virsh vol-delete --pool default
-vagrant box list | grep virtualbox | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider virtualbox --force
-vagrant box list | grep libvirt | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider libvirt --force

distros/almalinux/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "8";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "9";
+	arch = "amd64";
+	eol = false;
+} ]

distros/almalinux/supports.yml

@@ -1,9 +0,0 @@
-9: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/alpine/default.nix

@@ -0,0 +1,5 @@
+[ {
+	version = "3.19.0";
+	arch = "amd64";
+	eol = false;
+} ]

distros/alpine/supports.yml

@@ -1,7 +0,0 @@
-"3.19.0":
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/archlinux/default.nix

@@ -0,0 +1,5 @@
+[ {
+	version = "rolling";
+	arch = "amd64";
+	eol = false;
+} ]

distros/archlinux/supports.yml

@@ -1,9 +0,0 @@
-rolling:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/centos-stream/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "8";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "9";
+	arch = "amd64";
+	eol = false;
+} ]

distros/centos-stream/supports.yml

@@ -1,9 +0,0 @@
-9: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/centos/default.nix

@@ -0,0 +1,5 @@
+[ {
+	version = "7";
+	arch = "amd64";
+	eol = false;
+} ]

distros/centos/supports.yml

@@ -1,9 +0,0 @@
-7:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/debian/default.nix

@@ -0,0 +1,20 @@
+[ {
+	version = "9";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "10";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "11";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "12";
+	arch = "amd64";
+	eol = false;
+	exclude = [
+		"virtualbox-iso"
+	];
+} ]

distros/debian/supports.yml

@@ -1,17 +0,0 @@
-10: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-11: *supp
-12:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/default.nix

@@ -0,0 +1,13 @@
+{
+	almalinux = import ./almalinux;
+	alpine = import ./alpine;
+	archlinux = import ./archlinux;
+	centos = import ./centos;
+	centos-stream = import ./centos-stream;
+	debian = import ./debian;
+	fedora = import ./fedora;
+	nixos = import ./nixos;
+	opensuse = import ./opensuse;
+	rockylinux = import ./rockylinux;
+	ubuntu = import ./ubuntu;
+}

distros/fedora/default.nix

@@ -0,0 +1,21 @@
+[ {
+	version = "37";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "38";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "39";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "40";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "rawhide";
+	arch = "amd64";
+	eol = false;
+} ]

distros/fedora/supports.yml

@@ -1,21 +0,0 @@
-37:  # EOL
-38:  # EOL
-39: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-40: *supp
-rawhide:
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: hyperv-iso
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso

distros/nixos/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "23.05";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "23.11";
+	arch = "amd64";
+	eol = false;
+} ]

distros/nixos/supports.yml

@@ -1,10 +0,0 @@
-23.05:  # EOL
-23.11: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/opensuse/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "15.5";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "tumbleweed";
+	arch = "amd64";
+	eol = false;
+} ]

distros/opensuse/supports.yml

@@ -1,11 +0,0 @@
-15.5: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-15.6:
-tumbleweed: *supp

distros/rockylinux/default.nix

@@ -0,0 +1,9 @@
+[ {
+	version = "8";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "9";
+	arch = "amd64";
+	eol = false;
+} ]

distros/rockylinux/supports.yml

@@ -1,9 +0,0 @@
-9: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso

distros/ubuntu/default.nix

@@ -0,0 +1,17 @@
+[ {
+	version = "18.04";
+	arch = "amd64";
+	eol = true;
+} {
+	version = "20.04";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "22.04";
+	arch = "amd64";
+	eol = false;
+} {
+	version = "23.10";
+	arch = "amd64";
+	eol = false;
+} ]

distros/ubuntu/supports.yml

@@ -1,13 +0,0 @@
-20.04: &supp
-  - arch: amd64
-    provider: qemu
-  - arch: amd64
-    provider: virtualbox-iso
-  - arch: amd64
-    provider: vmware-iso
-  - arch: amd64
-    provider: hyperv-iso
-  # linux-azure package broken in 20.04
-22.04: *supp
-"23.10": *supp
-#18.04: *supp  # EOL

flake.nix

@@ -10,8 +10,30 @@
 		treefmt-nix.url = "github:numtide/treefmt-nix";
 	};
 
-	outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, ... }@inputs:
+	outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, nixpkgs, ... }@inputs: let
-	flake-parts.lib.mkFlake { inherit inputs; } {
+		lib = nixpkgs.lib;
+		providers = [
+			"qemu"
+			"virtualbox-iso"
+			"hyperv-iso"
+		];
+		distros = import ./distros;
+		# Inserts the distro name into each element in the list
+		insertNames = lib.attrsets.mapAttrsToList (distro: val: (builtins.map (x: x // { inherit distro; }) val)) distros;
+		# Flattens all the distros into a single list
+		flat = lib.lists.flatten insertNames;
+		# Combines providers list with flattened list
+		withProviders = lib.attrsets.mapCartesianProduct
+			({d, provider}: d // { inherit provider; })
+			{ d = flat; provider = providers; };
+		active = builtins.filter (x: (x?eol) && (!x.eol)) withProviders;
+		name = d: "-${d.distro}-${d.arch}-${d.provider}-${d.version}";
+		distroFile = d: "distros/${d.distro}/${d.version}/${d.arch}.pkrvars.hcl";
+		cmd = d: ''
+			version="''${1:-1}"
+			packer build -var "build=$version" -only=${d.provider}.${d.arch} -var-file=${distroFile d}'';
+	in flake-parts.lib.mkFlake { inherit inputs; }
+	{
 		imports = [
 			inputs.devshell.flakeModule
 			inputs.flake-root.flakeModule
@@ -24,9 +46,8 @@
 			"x86_64-linux"
 			"aarch64-linux"
 		];
-		perSystem = { config, pkgs, self', system, ... }: let
+		perSystem = { config, pkgs, self', system, ... }:
-			lib = pkgs.lib;
+		{
-		in {
 			_module.args.pkgs = import inputs.nixpkgs {
 				inherit system;
 				config.allowUnfree = true;
@@ -47,11 +68,46 @@
 					package = self'.packages.validate;
 					category = "utils";
 					help = "Validate all the files";
-				} ];
+				} ] ++ (builtins.map (d: { package = self'.packages."build${name d}"; category = "builds"; }) withProviders);
+				env = [{
+					name = "EFI_DIR";
+					value = "${pkgs.OVMF.fd}/FV/";
+				}];
 			};
 
 			# This is not special, this is just flakes
-			packages = {
+			packages = let
+				base = {
+					runtimeInputs = with pkgs; [
+						packer
+						qemu_full
+					];
+				};
+				build = d: pkgs.writeShellApplication (base // {
+					name = "build${name d}";
+					text = ''
+						set -x
+						${cmd d} -except=upload -var headless=false sources/
+					'';
+				});
+				upload = d: pkgs.writeShellApplication (base // {
+					name = "upload${name d}";
+					text = ''
+						set -x
+						${cmd d} sources/
+					'';
+				});
+				builds = lib.lists.foldr (d: acc: acc // { "build${name d}" = (build d); }) {} withProviders;
+				uploads = lib.lists.foldr (d: acc: acc // { "upload${name d}" = (upload d); }) {} withProviders;
+				toYAML = x: (pkgs.formats.yaml x).generate;
+				qemu_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "qemu") active));
+				vbox_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "virtualbox-iso") active));
+				hyperv_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "hyperv-iso") active));
+			in builds // uploads // {
+				continue = pkgs.writeText "continue.yml"
+					(builtins.replaceStrings ["QEMU_MATRIX" "VBOX_MATRIX" "HYPERV_MATRIX"] [qemu_list vbox_list hyperv_list]
+						(builtins.readFile ./.gitlab-ci-build.yml.in)
+					);
 				cache = pkgs.writeShellApplication {
 					name = "cache-isos";
 					runtimeInputs = with pkgs; [

lib/build.py

@@ -1,40 +0,0 @@
-from provider import Provider
-
-
-class Build:
-    def __init__(self, distro, version, provider: Provider):
-        self.distro: str = distro
-        self.version: str = str(version)
-        self.provider: Provider = provider
-
-    @property
-    def var_file(self):
-        return f"-var-file={self.var_path}"
-
-    @property
-    def matrix(self):
-        return {
-            "builder": str(self.provider),
-            "distro": self.var_path,
-            "runner": self.provider.runner,
-        }
-
-    @property
-    def is_github(self) -> bool:
-        if self.provider.runner is not None:
-            return True
-        return False
-
-    @property
-    def var_path(self):
-        return f"distros/{self.distro}/{self.version}/{self.provider.arch}.pkrvars.hcl"
-
-    @property
-    def only(self):
-        return f"-only={self.provider}"
-
-    def __str__(self) -> str:
-        return f"{self.distro} {self.version}: {self.provider}"
-
-    def __repr__(self) -> str:
-        return str(self)

lib/distro.py

@@ -1,28 +0,0 @@
-from provider import Provider
-from build import Build
-
-
-class Distro:
-    def __init__(self, name: str, support: map):
-        self.name = name
-        self.builds = []
-        for version, matrix in support.items():
-            if not matrix:
-                continue
-            for support in matrix:
-                provider = Provider(support["provider"], support["arch"])
-                build = Build(self.name, version, provider)
-                self.builds.append(build)
-
-    def __str__(self) -> str:
-        return f"{self.name}"
-
-    def __repr__(self) -> str:
-        return f"{self.name}: {' '.join(self.versions)}"
-
-    @property
-    def versions(self) -> set[str]:
-        return set([b.version for b in self.builds])
-
-    def builds_for_version(self, version: str) -> set[Build]:
-        return set(filter(lambda x: x.version == version, self.builds))

lib/provider.py

@@ -1,17 +0,0 @@
-class Provider:
-    def __init__(self, name, arch):
-        self.name = name
-        self.arch = arch
-
-    def __str__(self) -> str:
-        return f"{self.name}.{self.arch}"
-
-    @property
-    def runner(self) -> str:
-        if self.name == "qemu":
-            return "qemu"
-        elif self.name.startswith("virtualbox"):
-            return "virtualbox"
-        elif self.name.startswith("hyperv"):
-            return "hyperv"
-        return None

lib/support.py

@@ -1,23 +0,0 @@
-from distro import Distro
-from provider import Provider
-from pathlib import Path
-from yaml import load, Loader
-
-
-BASE = Path(__file__).parent.parent.resolve()
-
-
-class Support:
-    def __init__(self):
-        self.base = BASE
-        self.distros = self.__get_distros()
-
-    def __get_distros(self) -> map:
-        distros = {}
-        distros_dir = self.base / "distros"
-        for f in distros_dir.glob("**/supports.yml"):
-            name = f.parent.name
-            with open(f, "r") as fp:
-                support_yml = load(fp, Loader=Loader)
-            distros[name] = Distro(name, support_yml)
-        return distros

sources/inputs.pkr.hcl

@@ -109,6 +109,16 @@ variable "vbox_os_type" {
   default = "Linux_64"
 }
 
+variable "storage_url" {
+  type    = string
+  default = env("STORAGE_URL")
+}
+
+variable "minio_secret" {
+  type    = string
+  default = env("MINIO_SECRET")
+}
+
 locals {
   name = "${var.distro}-${var.version}"
   description = templatefile("../README.box.md", {

sources/qemu.pkr.hcl

@@ -4,7 +4,7 @@ variable "qemu_accelerator" {
 }
 
 source "qemu" "amd64" {
-  iso_urls         = [ "cache/${basename(var.iso.url)}" ]
+  iso_urls         = [ "s3::${var.storage_url}isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ]
   iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/libvirt/${local.name}"

test.xsh

@@ -1,88 +0,0 @@
-#!env xonsh
-# vim: set ft=xonsh:
-import argparse
-import pathlib
-import re
-import sys
-import time
-from shutil import rmtree
-
-
-def get_vagrant_provider(provider):
-    "Vagrant provider is not always the same as Packer provider"
-    if provider.startswith("qemu"):
-        return "libvirt"
-    elif provider.startswith("virtualbox"):
-        return "virtualbox"
-    elif provider.startswith("vmware"):
-        return "vmware"
-    elif provider.startswith("hyperv"):
-        return "hyperv"
-    else:
-        return provider
-
-
-def get_providers():
-    "List of providers as defined in sources/"
-    providers: set[str] = {
-        "qemu.amd64"
-        "virtualbox-iso.amd64",
-        "vmware-iso.amd64"
-    }
-    return providers
-
-
-def get_parser(args, unsupported=set()):
-    parser = argparse.ArgumentParser("Builder")
-    parser.add_argument("--headless", action="store_true")
-    parser.add_argument("--upload", action="store_true")
-    parser.add_argument("--provider", action="append")
-    parsed = parser.parse_args(args)
-    opts = {}
-    if parsed.headless:
-        opts["headless"] = "headless=true"
-    else:
-        opts["headless"] = "headless=false"
-    if parsed.upload:
-        opts["upload"] = ""
-    else:
-        opts["upload"] = "-except=upload"
-    if parsed.provider:
-        opts["providers"] = set(parsed.provider)
-    else:
-        opts["providers"] = get_providers()
-    # Filter providers
-    opts["providers"] -= unsupported
-    return opts
-
-
-def main():
-    # Equivalent of `set -e` in Bash
-    $RAISE_SUBPROC_ERROR = True
-    if len(sys.argv) == 2:
-        dry_run = True
-    else:
-        dry_run = False
-    for box in g`output/**/*.box`:
-        p = pathlib.Path(box)
-        with open("Vagrantfile.in", "r") as fr:
-            content = fr.read()
-        content = content.replace("@@BOX@@", p.resolve().as_uri())
-        content = content.replace("@@BOX_NAME@@", p.name)
-        with open("Vagrantfile", "w") as fw:
-            fw.write(content)
-        if not dry_run:
-            if str(p.parent.name) in ["virtualbox", "libvirt", "vmware", "hyperv"]:
-                vagrant --provider @(p.parent.name) box add @(box) --name @(p.name) --force
-                vagrant up --provider @(p.parent.name)
-            else:
-                print(f"Now whatcha wanna go and do that for? {box}")
-                continue
-            vagrant destroy -f
-            vag_folder = pathlib.Path(".vagrant")
-            if vag_folder.exists():
-                rmtree(vag_folder)
-
-
-if __name__ == '__main__':
-    main()