Move almost everything over to Nix

Eliminate Xonsh usage
Eliminate usage of the supports.yml files for default.nix
Point ISOs directly to MinIO URLs instead
This commit is contained in:
Greg Hellings
2024-06-26 16:00:57 -05:00
parent 2d124c652a
commit 832c901b9b
37 changed files with 210 additions and 434 deletions
+1
View File
@@ -8,6 +8,7 @@ grub.cfg
isolinux.cfg
Vagrantfile
.direnv
result
# Generated distros
distros/fedora/rawhide/*.hcl
+3 -6
View File
@@ -39,10 +39,7 @@ stages:
-var memory=4096
${EXCEPT}
sources
before_script:
- nix run ".#cache-restore"
script:
- ls -l cache/
- echo $COMMAND > build.sh
- bash build.sh || exit 1
@@ -53,7 +50,7 @@ qemu.amd64:
only: "-only=qemu.amd64"
tag: qemu
parallel:
matrix: []
matrix: QEMU_MATRIX
virtualbox-iso.amd64:
extends: .build
@@ -62,7 +59,7 @@ virtualbox-iso.amd64:
only: '-only="virtualbox-iso.amd64"'
tag: vbox
parallel:
matrix: []
matrix: VBOX_MATRIX
hyperv-iso.amd64:
extends: .build
@@ -71,7 +68,7 @@ hyperv-iso.amd64:
only: '-only="hyperv-iso.amd64"'
tag: hyperv
parallel:
matrix: []
matrix: HYPERV_MATRIX
script:
- echo $env:COMMAND > build.ps1
- echo 'if (-not $?) { throw "Error in build"}' >> build.ps1
-93
View File
@@ -1,93 +0,0 @@
#!/usr/bin/env xonsh
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).parent / "lib"))
from support import Support, BASE
import time
import json
from argparse import ArgumentParser
from functools import reduce
from yaml import dump, Dumper, load, Loader
source test.xsh
year, month, day, hour, minute, second, wday, jday, dst = time.localtime()
STAMP = f"build={year}.{month}.{day}.{hour}"
support = Support()
parser = ArgumentParser("Build a box")
parser.add_argument("--distro", "-d", help="Name of the distro/ folder")
parser.add_argument("--version", "-v", help="Version of the distro to build")
parser.add_argument("--build", "-b", help="Name of the target build, e.g. qemu.amd64, virtualbox-iso.amd64")
parser.add_argument("--list", "-l", help="List all build targets", action="store_true")
parser.add_argument("--all", "-a", help="Build all applicable targets", action="store_true")
parser.add_argument("--upload", "-u", help="Force doing uploads", action="store_true")
parser.add_argument("--headless", help="Run builds headless", action="store_true")
parser.add_argument("--stamp", "-s", help="Time stamp for builds", default=STAMP)
args = parser.parse_args()
my_list = []
# Now provide menus if we don't have defaults
def process_all():
if args.distro:
process_distro(support.distros[args.distro])
else:
for name, distro in support.distros.items():
process_distro(distro)
def process_distro(distro):
if args.version:
process_builds(distro.builds_for_version(args.version))
else:
process_builds(distro.builds)
def process_builds(builds):
for build in builds:
if args.build:
name, arch = args.build.split(".")
if build.provider.name == name and build.provider.arch == arch:
do_build(build)
else:
do_build(build)
def do_build(build):
if args.headless:
headless = "headless=true"
else:
headless = "headless=false"
if args.list:
my_list.append(build)
else:
if args.upload:
if "VAGRANT_CLOUD_TOKEN" not in ${...}:
print("You need to set your cloud token")
sys.exit(1)
packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) @(BASE / "sources")
else:
upload = "-except=upload"
# Skip building file if we already have built it
output = Path(__file__).parent / "output" / get_vagrant_provider(build.provider.name) / f"{build.distro}-{build.version}-{build.provider.arch}.box"
if output.exists():
print(f"Skipping {build} as box already exists")
else:
packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) -except=upload @(BASE / "sources")
if args.all or args.list or (args.distro and args.version and args.build):
process_all()
if args.list:
with open(".gitlab-ci-build.yml.in", "r") as fp:
y = load(fp, Loader=Loader)
for item in my_list:
print(item)
if item.is_github:
y[ item.only[6:] ]["parallel"]["matrix"].append({"distro": item.var_path})
with open(".gitlab-ci-build.yml", "w") as fp:
dump(y, fp, Dumper=Dumper)
else:
print("Doing menu stuff")
+1 -1
View File
@@ -4,7 +4,7 @@ set -eo pipefail
files=()
# These files are always out of date
rm -f cache/*latest* cache/*Rawhide*
rm -f cache/*latest* cache/*Rawhide* cache/openSUSE*Current*
# shellcheck disable=SC2044
for vars in $(find distros/ -name '*.pkrvars.hcl'); do
-6
View File
@@ -1,6 +0,0 @@
#!/usr/bin/env bash
virsh -q vol-list default | awk '{print $1}' | xargs -r -L 1 virsh vol-delete --pool default
sudo virsh -q vol-list default | awk '{print $1}' | sudo xargs -r -L 1 virsh vol-delete --pool default
vagrant box list | grep virtualbox | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider virtualbox --force
vagrant box list | grep libvirt | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider libvirt --force
+9
View File
@@ -0,0 +1,9 @@
[ {
version = "8";
arch = "amd64";
eol = true;
} {
version = "9";
arch = "amd64";
eol = false;
} ]
-9
View File
@@ -1,9 +0,0 @@
9: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+5
View File
@@ -0,0 +1,5 @@
[ {
version = "3.19.0";
arch = "amd64";
eol = false;
} ]
-7
View File
@@ -1,7 +0,0 @@
"3.19.0":
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: hyperv-iso
+5
View File
@@ -0,0 +1,5 @@
[ {
version = "rolling";
arch = "amd64";
eol = false;
} ]
-9
View File
@@ -1,9 +0,0 @@
rolling:
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+9
View File
@@ -0,0 +1,9 @@
[ {
version = "8";
arch = "amd64";
eol = true;
} {
version = "9";
arch = "amd64";
eol = false;
} ]
-9
View File
@@ -1,9 +0,0 @@
9: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+5
View File
@@ -0,0 +1,5 @@
[ {
version = "7";
arch = "amd64";
eol = false;
} ]
-9
View File
@@ -1,9 +0,0 @@
7:
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+20
View File
@@ -0,0 +1,20 @@
[ {
version = "9";
arch = "amd64";
eol = true;
} {
version = "10";
arch = "amd64";
eol = false;
} {
version = "11";
arch = "amd64";
eol = false;
} {
version = "12";
arch = "amd64";
eol = false;
exclude = [
"virtualbox-iso"
];
} ]
-17
View File
@@ -1,17 +0,0 @@
10: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
11: *supp
12:
- arch: amd64
provider: qemu
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+13
View File
@@ -0,0 +1,13 @@
{
almalinux = import ./almalinux;
alpine = import ./alpine;
archlinux = import ./archlinux;
centos = import ./centos;
centos-stream = import ./centos-stream;
debian = import ./debian;
fedora = import ./fedora;
nixos = import ./nixos;
opensuse = import ./opensuse;
rockylinux = import ./rockylinux;
ubuntu = import ./ubuntu;
}
+21
View File
@@ -0,0 +1,21 @@
[ {
version = "37";
arch = "amd64";
eol = true;
} {
version = "38";
arch = "amd64";
eol = true;
} {
version = "39";
arch = "amd64";
eol = false;
} {
version = "40";
arch = "amd64";
eol = false;
} {
version = "rawhide";
arch = "amd64";
eol = false;
} ]
-21
View File
@@ -1,21 +0,0 @@
37: # EOL
38: # EOL
39: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
40: *supp
rawhide:
- arch: amd64
provider: qemu
- arch: amd64
provider: hyperv-iso
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
+9
View File
@@ -0,0 +1,9 @@
[ {
version = "23.05";
arch = "amd64";
eol = true;
} {
version = "23.11";
arch = "amd64";
eol = false;
} ]
-10
View File
@@ -1,10 +0,0 @@
23.05: # EOL
23.11: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+9
View File
@@ -0,0 +1,9 @@
[ {
version = "15.5";
arch = "amd64";
eol = false;
} {
version = "tumbleweed";
arch = "amd64";
eol = false;
} ]
-11
View File
@@ -1,11 +0,0 @@
15.5: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
15.6:
tumbleweed: *supp
+9
View File
@@ -0,0 +1,9 @@
[ {
version = "8";
arch = "amd64";
eol = true;
} {
version = "9";
arch = "amd64";
eol = false;
} ]
-9
View File
@@ -1,9 +0,0 @@
9: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
+17
View File
@@ -0,0 +1,17 @@
[ {
version = "18.04";
arch = "amd64";
eol = true;
} {
version = "20.04";
arch = "amd64";
eol = false;
} {
version = "22.04";
arch = "amd64";
eol = false;
} {
version = "23.10";
arch = "amd64";
eol = false;
} ]
-13
View File
@@ -1,13 +0,0 @@
20.04: &supp
- arch: amd64
provider: qemu
- arch: amd64
provider: virtualbox-iso
- arch: amd64
provider: vmware-iso
- arch: amd64
provider: hyperv-iso
# linux-azure package broken in 20.04
22.04: *supp
"23.10": *supp
#18.04: *supp # EOL
+63 -7
View File
@@ -10,8 +10,30 @@
treefmt-nix.url = "github:numtide/treefmt-nix";
};
outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, ... }@inputs:
flake-parts.lib.mkFlake { inherit inputs; } {
outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, nixpkgs, ... }@inputs: let
lib = nixpkgs.lib;
providers = [
"qemu"
"virtualbox-iso"
"hyperv-iso"
];
distros = import ./distros;
# Inserts the distro name into each element in the list
insertNames = lib.attrsets.mapAttrsToList (distro: val: (builtins.map (x: x // { inherit distro; }) val)) distros;
# Flattens all the distros into a single list
flat = lib.lists.flatten insertNames;
# Combines providers list with flattened list
withProviders = lib.attrsets.mapCartesianProduct
({d, provider}: d // { inherit provider; })
{ d = flat; provider = providers; };
active = builtins.filter (x: (x?eol) && (!x.eol)) withProviders;
name = d: "-${d.distro}-${d.arch}-${d.provider}-${d.version}";
distroFile = d: "distros/${d.distro}/${d.version}/${d.arch}.pkrvars.hcl";
cmd = d: ''
version="''${1:-1}"
packer build -var "build=$version" -only=${d.provider}.${d.arch} -var-file=${distroFile d}'';
in flake-parts.lib.mkFlake { inherit inputs; }
{
imports = [
inputs.devshell.flakeModule
inputs.flake-root.flakeModule
@@ -24,9 +46,8 @@
"x86_64-linux"
"aarch64-linux"
];
perSystem = { config, pkgs, self', system, ... }: let
lib = pkgs.lib;
in {
perSystem = { config, pkgs, self', system, ... }:
{
_module.args.pkgs = import inputs.nixpkgs {
inherit system;
config.allowUnfree = true;
@@ -47,11 +68,46 @@
package = self'.packages.validate;
category = "utils";
help = "Validate all the files";
} ];
} ] ++ (builtins.map (d: { package = self'.packages."build${name d}"; category = "builds"; }) withProviders);
env = [{
name = "EFI_DIR";
value = "${pkgs.OVMF.fd}/FV/";
}];
};
# This is not special, this is just flakes
packages = {
packages = let
base = {
runtimeInputs = with pkgs; [
packer
qemu_full
];
};
build = d: pkgs.writeShellApplication (base // {
name = "build${name d}";
text = ''
set -x
${cmd d} -except=upload -var headless=false sources/
'';
});
upload = d: pkgs.writeShellApplication (base // {
name = "upload${name d}";
text = ''
set -x
${cmd d} sources/
'';
});
builds = lib.lists.foldr (d: acc: acc // { "build${name d}" = (build d); }) {} withProviders;
uploads = lib.lists.foldr (d: acc: acc // { "upload${name d}" = (upload d); }) {} withProviders;
toYAML = x: (pkgs.formats.yaml x).generate;
qemu_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "qemu") active));
vbox_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "virtualbox-iso") active));
hyperv_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "hyperv-iso") active));
in builds // uploads // {
continue = pkgs.writeText "continue.yml"
(builtins.replaceStrings ["QEMU_MATRIX" "VBOX_MATRIX" "HYPERV_MATRIX"] [qemu_list vbox_list hyperv_list]
(builtins.readFile ./.gitlab-ci-build.yml.in)
);
cache = pkgs.writeShellApplication {
name = "cache-isos";
runtimeInputs = with pkgs; [
View File
-40
View File
@@ -1,40 +0,0 @@
from provider import Provider
class Build:
def __init__(self, distro, version, provider: Provider):
self.distro: str = distro
self.version: str = str(version)
self.provider: Provider = provider
@property
def var_file(self):
return f"-var-file={self.var_path}"
@property
def matrix(self):
return {
"builder": str(self.provider),
"distro": self.var_path,
"runner": self.provider.runner,
}
@property
def is_github(self) -> bool:
if self.provider.runner is not None:
return True
return False
@property
def var_path(self):
return f"distros/{self.distro}/{self.version}/{self.provider.arch}.pkrvars.hcl"
@property
def only(self):
return f"-only={self.provider}"
def __str__(self) -> str:
return f"{self.distro} {self.version}: {self.provider}"
def __repr__(self) -> str:
return str(self)
-28
View File
@@ -1,28 +0,0 @@
from provider import Provider
from build import Build
class Distro:
def __init__(self, name: str, support: map):
self.name = name
self.builds = []
for version, matrix in support.items():
if not matrix:
continue
for support in matrix:
provider = Provider(support["provider"], support["arch"])
build = Build(self.name, version, provider)
self.builds.append(build)
def __str__(self) -> str:
return f"{self.name}"
def __repr__(self) -> str:
return f"{self.name}: {' '.join(self.versions)}"
@property
def versions(self) -> set[str]:
return set([b.version for b in self.builds])
def builds_for_version(self, version: str) -> set[Build]:
return set(filter(lambda x: x.version == version, self.builds))
-17
View File
@@ -1,17 +0,0 @@
class Provider:
def __init__(self, name, arch):
self.name = name
self.arch = arch
def __str__(self) -> str:
return f"{self.name}.{self.arch}"
@property
def runner(self) -> str:
if self.name == "qemu":
return "qemu"
elif self.name.startswith("virtualbox"):
return "virtualbox"
elif self.name.startswith("hyperv"):
return "hyperv"
return None
-23
View File
@@ -1,23 +0,0 @@
from distro import Distro
from provider import Provider
from pathlib import Path
from yaml import load, Loader
BASE = Path(__file__).parent.parent.resolve()
class Support:
def __init__(self):
self.base = BASE
self.distros = self.__get_distros()
def __get_distros(self) -> map:
distros = {}
distros_dir = self.base / "distros"
for f in distros_dir.glob("**/supports.yml"):
name = f.parent.name
with open(f, "r") as fp:
support_yml = load(fp, Loader=Loader)
distros[name] = Distro(name, support_yml)
return distros
+10
View File
@@ -109,6 +109,16 @@ variable "vbox_os_type" {
default = "Linux_64"
}
variable "storage_url" {
type = string
default = env("STORAGE_URL")
}
variable "minio_secret" {
type = string
default = env("MINIO_SECRET")
}
locals {
name = "${var.distro}-${var.version}"
description = templatefile("../README.box.md", {
+1 -1
View File
@@ -4,7 +4,7 @@ variable "qemu_accelerator" {
}
source "qemu" "amd64" {
iso_urls = [ "cache/${basename(var.iso.url)}" ]
iso_urls = [ "s3::${var.storage_url}isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ]
iso_target_path = "cache/${basename(var.iso.url)}"
iso_checksum = var.iso.checksum
output_directory = "output/libvirt/${local.name}"
-88
View File
@@ -1,88 +0,0 @@
#!env xonsh
# vim: set ft=xonsh:
import argparse
import pathlib
import re
import sys
import time
from shutil import rmtree
def get_vagrant_provider(provider):
"Vagrant provider is not always the same as Packer provider"
if provider.startswith("qemu"):
return "libvirt"
elif provider.startswith("virtualbox"):
return "virtualbox"
elif provider.startswith("vmware"):
return "vmware"
elif provider.startswith("hyperv"):
return "hyperv"
else:
return provider
def get_providers():
"List of providers as defined in sources/"
providers: set[str] = {
"qemu.amd64"
"virtualbox-iso.amd64",
"vmware-iso.amd64"
}
return providers
def get_parser(args, unsupported=set()):
parser = argparse.ArgumentParser("Builder")
parser.add_argument("--headless", action="store_true")
parser.add_argument("--upload", action="store_true")
parser.add_argument("--provider", action="append")
parsed = parser.parse_args(args)
opts = {}
if parsed.headless:
opts["headless"] = "headless=true"
else:
opts["headless"] = "headless=false"
if parsed.upload:
opts["upload"] = ""
else:
opts["upload"] = "-except=upload"
if parsed.provider:
opts["providers"] = set(parsed.provider)
else:
opts["providers"] = get_providers()
# Filter providers
opts["providers"] -= unsupported
return opts
def main():
# Equivalent of `set -e` in Bash
$RAISE_SUBPROC_ERROR = True
if len(sys.argv) == 2:
dry_run = True
else:
dry_run = False
for box in g`output/**/*.box`:
p = pathlib.Path(box)
with open("Vagrantfile.in", "r") as fr:
content = fr.read()
content = content.replace("@@BOX@@", p.resolve().as_uri())
content = content.replace("@@BOX_NAME@@", p.name)
with open("Vagrantfile", "w") as fw:
fw.write(content)
if not dry_run:
if str(p.parent.name) in ["virtualbox", "libvirt", "vmware", "hyperv"]:
vagrant --provider @(p.parent.name) box add @(box) --name @(p.name) --force
vagrant up --provider @(p.parent.name)
else:
print(f"Now whatcha wanna go and do that for? {box}")
continue
vagrant destroy -f
vag_folder = pathlib.Path(".vagrant")
if vag_folder.exists():
rmtree(vag_folder)
if __name__ == '__main__':
main()