diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index 5012c47..bbe3246 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ serial-output* grub.cfg isolinux.cfg Vagrantfile +.direnv +result # Generated distros distros/fedora/rawhide/*.hcl diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 6da64c2..cf9f0cf 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -11,9 +11,6 @@ stages: - sources/build.pkr.hcl paths: - ./packer_config/ - - key: isofiles - paths: - - cache/*.iso rules: - if: $CI_COMMIT_TAG variables: @@ -23,7 +20,7 @@ stages: EXCEPT: '-except=upload' needs: - pipeline: $PARENT_PIPELINE_ID - job: distros + job: continue artifacts: true - pipeline: $PARENT_PIPELINE_ID job: generate @@ -42,9 +39,6 @@ stages: -var memory=4096 ${EXCEPT} sources - before_script: - - packer init sources - - echo $COMMAND script: - echo $COMMAND > build.sh - bash build.sh || exit 1 @@ -56,7 +50,7 @@ qemu.amd64: only: "-only=qemu.amd64" tag: qemu parallel: - matrix: [] + matrix: QEMU_MATRIX virtualbox-iso.amd64: extends: .build @@ -65,11 +59,7 @@ virtualbox-iso.amd64: only: '-only="virtualbox-iso.amd64"' tag: vbox parallel: - matrix: [] - after_script: |- - if [ "$CI_JOB_STATUS" == "success" ]; then - xonsh test.xsh - fi + matrix: VBOX_MATRIX hyperv-iso.amd64: extends: .build @@ -78,15 +68,11 @@ hyperv-iso.amd64: only: '-only="hyperv-iso.amd64"' tag: hyperv parallel: - matrix: [] + matrix: HYPERV_MATRIX script: - echo $env:COMMAND > build.ps1 - echo 'if (-not $?) { throw "Error in build"}' >> build.ps1 - pwsh .\build.ps1 - after_script: |- - if ( "$CI_JOB_STATUS" -eq "success" ) { - xonsh test.xsh - } "Create release": stage: release @@ -96,7 +82,7 @@ hyperv-iso.amd64: script: echo "Tagging as released" needs: - pipeline: $PARENT_PIPELINE_ID - job: distros + job: continue artifacts: true release: tag_name: '$CI_COMMIT_TAG' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9bbd705..5ce6a85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,41 +1,42 @@ stages: - generate - - lint - - cache + - check - trigger variables: PACKER_CONFIG_DIR: ./packer_config/ PACKER_CACHE_DIR: ./cache/ +default: + image: registry.thehellings.com/greg/ci-images/vm-test:latest + tags: + - nix + generate: stage: generate - cache: - - key: isofiles - paths: - - cache/*.iso artifacts: paths: - distros/**/*.pkrvars.hcl - script: |- - for vars in $(find distros/ -name '*.pkrvars.hcl.sh'); do - "./${vars}" - done + script: nix run ".#generate" -distros: +continue: stage: generate - image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest artifacts: reports: dotenv: build.env paths: - .gitlab-ci-build.yml - script: |- - xonsh ./build.xsh -a --list - echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env + script: + - nix build ".#continue" + - cp result .gitlab-ci-build.yml + - cat result + +shellcheck: + stage: generate + script: nix run ".#shellcheck" validate: - stage: lint + stage: check needs: - generate cache: @@ -44,45 +45,28 @@ validate: - sources/build.pkr.hcl paths: - ./packer_config/ - script: |- - set -exo pipefail - packer init sources - for vars in $(find distros/ -name '*.pkrvars.hcl'); do - packer validate -except=upload -var-file=${vars} sources/ - done + script: nix run ".#validate" -shellcheck: - stage: lint - script: |- - shellcheck --version - shellcheck $(find . -name '*.sh') - -# Downloads all the ISO files so that we have a single cache with all -# of them present. This prevents a race condition where some of the -# files were being downloaded multiple times due to parallel jobs, and -# others were disappearing (e.g. daily builds for Rawhide) before we -# could get to them +# Makes sure that all of the ISO files we are looking for are +# present on the S3 buckets that we use downloads: - stage: cache - image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest + stage: check + retry: 2 + needs: + - generate cache: - - key: isofiles - paths: - - cache/*.iso - key: files: - sources/build.pkr.hcl paths: - ./packer_config/ - script: - - packer init sources - - bash cache.sh + script: nix run ".#cache" trigger: stage: trigger trigger: include: - artifact: .gitlab-ci-build.yml - job: distros + job: continue variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID diff --git a/build.xsh b/build.xsh deleted file mode 100755 index d129fe7..0000000 --- a/build.xsh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env xonsh -from pathlib import Path -import sys -sys.path.insert(0, str(Path(__file__).parent / "lib")) -from support import Support, BASE -import time -import json - -from argparse import ArgumentParser -from functools import reduce -from yaml import dump, Dumper, load, Loader - -source test.xsh - - -year, month, day, hour, minute, second, wday, jday, dst = time.localtime() -STAMP = f"build={year}.{month}.{day}.{hour}" - -support = Support() -parser = ArgumentParser("Build a box") -parser.add_argument("--distro", "-d", help="Name of the distro/ folder") -parser.add_argument("--version", "-v", help="Version of the distro to build") -parser.add_argument("--build", "-b", help="Name of the target build, e.g. qemu.amd64, virtualbox-iso.amd64") -parser.add_argument("--list", "-l", help="List all build targets", action="store_true") -parser.add_argument("--all", "-a", help="Build all applicable targets", action="store_true") -parser.add_argument("--upload", "-u", help="Force doing uploads", action="store_true") -parser.add_argument("--headless", help="Run builds headless", action="store_true") -parser.add_argument("--stamp", "-s", help="Time stamp for builds", default=STAMP) -args = parser.parse_args() - - -my_list = [] - - -# Now provide menus if we don't have defaults -def process_all(): - if args.distro: - process_distro(support.distros[args.distro]) - else: - for name, distro in support.distros.items(): - process_distro(distro) - -def process_distro(distro): - if args.version: - process_builds(distro.builds_for_version(args.version)) - else: - process_builds(distro.builds) - -def process_builds(builds): - for build in builds: - if args.build: - name, arch = args.build.split(".") - if build.provider.name == name and build.provider.arch == arch: - do_build(build) - else: - do_build(build) - -def do_build(build): - if args.headless: - headless = "headless=true" - else: - headless = "headless=false" - - if args.list: - my_list.append(build) - else: - if args.upload: - if "VAGRANT_CLOUD_TOKEN" not in ${...}: - print("You need to set your cloud token") - sys.exit(1) - packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) @(BASE / "sources") - else: - upload = "-except=upload" - # Skip building file if we already have built it - output = Path(__file__).parent / "output" / get_vagrant_provider(build.provider.name) / f"{build.distro}-{build.version}-{build.provider.arch}.box" - if output.exists(): - print(f"Skipping {build} as box already exists") - else: - packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) -except=upload @(BASE / "sources") - -if args.all or args.list or (args.distro and args.version and args.build): - process_all() - if args.list: - with open(".gitlab-ci-build.yml.in", "r") as fp: - y = load(fp, Loader=Loader) - for item in my_list: - print(item) - if item.is_github: - y[ item.only[6:] ]["parallel"]["matrix"].append({"distro": item.var_path}) - with open(".gitlab-ci-build.yml", "w") as fp: - dump(y, fp, Dumper=Dumper) -else: - print("Doing menu stuff") diff --git a/cache-restore.sh b/cache-restore.sh new file mode 100644 index 0000000..b5d8208 --- /dev/null +++ b/cache-restore.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + +for f in $(mc ls --json storage/isos/cache | jq '.key' | tr -d '"'); do + if [ ! -e "./cache/$f" ]; then + mc get -q "storage/isos/cache/$f" "./cache/$f" + else + echo "Skipping already present: $f" + fi +done diff --git a/cache.sh b/cache.sh deleted file mode 100755 index ab406a0..0000000 --- a/cache.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -files=() - -# shellcheck disable=SC2044 -for vars in $(find distros/ -name '*.pkrvars.hcl'); do - file=$(echo var.iso.url | packer console -var-file="${vars}" -config-type=hcl2 sources/) - base="$(basename "${file}")" - if [ -e "cache/${base}" ]; then - echo "Found cache/${base} already" - else - files+=("-O" "${file}") - fi -done - -mkdir -p cache -pushd cache -echo "Downloading ${files[*]}" -curl --parallel -L "${files[@]}" -popd diff --git a/del_boxes.xsh b/del_boxes.xsh deleted file mode 100755 index 79cb6ed..0000000 --- a/del_boxes.xsh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -virsh -q vol-list default | awk '{print $1}' | xargs -r -L 1 virsh vol-delete --pool default -sudo virsh -q vol-list default | awk '{print $1}' | sudo xargs -r -L 1 virsh vol-delete --pool default -vagrant box list | grep virtualbox | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider virtualbox --force -vagrant box list | grep libvirt | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider libvirt --force diff --git a/distros/almalinux/9/amd64.pkrvars.hcl b/distros/almalinux/9/amd64.pkrvars.hcl index 0aaffa1..235dc15 100644 --- a/distros/almalinux/9/amd64.pkrvars.hcl +++ b/distros/almalinux/9/amd64.pkrvars.hcl @@ -1,13 +1,14 @@ distro = "almalinux" version = "9" iso = { - url = "https://mirrors.kernel.org/almalinux/9.3/isos/x86_64/AlmaLinux-9.3-x86_64-boot.iso" - checksum = "af5377a1d16bbe599ea91a8761ad645f2f54687075802bdc0c0703ee610182e9" + url = "https://mirrors.kernel.org/almalinux/9.4/isos/x86_64/AlmaLinux-9.4-x86_64-boot.iso" + checksum = "1e5d7da3d84d5d9a5a1177858a5df21b868390bfccf7f0f419b1e59acc293160" } boot_command = [ "e", "", " inst.text inst.ks=cdrom:/ks.cfg", + " console=ttyS0 inst.notmux", "x" ] cd_files = [ "distros/almalinux/9/disc/*" ] diff --git a/distros/almalinux/default.nix b/distros/almalinux/default.nix new file mode 100644 index 0000000..a0837c6 --- /dev/null +++ b/distros/almalinux/default.nix @@ -0,0 +1,9 @@ +[ { + version = "8"; + arch = "amd64"; + eol = true; +} { + version = "9"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/almalinux/supports.yml b/distros/almalinux/supports.yml deleted file mode 100644 index 533b6df..0000000 --- a/distros/almalinux/supports.yml +++ /dev/null @@ -1,10 +0,0 @@ -8: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -9: *supp diff --git a/distros/alpine/3.19.0/amd64.pkrvars.hcl b/distros/alpine/3.19.0/amd64.pkrvars.hcl index 47936c3..894f4fd 100644 --- a/distros/alpine/3.19.0/amd64.pkrvars.hcl +++ b/distros/alpine/3.19.0/amd64.pkrvars.hcl @@ -16,4 +16,4 @@ boot_command = [ ] http_directory = "distros/alpine/3.19.0/http/" boot_wait = "20s" -shutdown_command = "sudo poweroff" \ No newline at end of file +shutdown_command = "sudo poweroff" diff --git a/distros/alpine/default.nix b/distros/alpine/default.nix new file mode 100644 index 0000000..15cdeb4 --- /dev/null +++ b/distros/alpine/default.nix @@ -0,0 +1,5 @@ +[ { + version = "3.19.0"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/alpine/supports.yml b/distros/alpine/supports.yml deleted file mode 100644 index 8c45e08..0000000 --- a/distros/alpine/supports.yml +++ /dev/null @@ -1,7 +0,0 @@ -"3.19.0": - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/archlinux/default.nix b/distros/archlinux/default.nix new file mode 100644 index 0000000..4fecf6e --- /dev/null +++ b/distros/archlinux/default.nix @@ -0,0 +1,5 @@ +[ { + version = "rolling"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/archlinux/rolling/amd64.pkrvars.hcl b/distros/archlinux/rolling/amd64.pkrvars.hcl index 6c0708c..60f9fdb 100644 --- a/distros/archlinux/rolling/amd64.pkrvars.hcl +++ b/distros/archlinux/rolling/amd64.pkrvars.hcl @@ -5,8 +5,8 @@ iso = { checksum = "52aea58f88c9a80afe64f0536da868251ef4878de5a5e0227fcada9f132bd7ab" } boot_command = [ - "e net.ifnames=0 biosdevnames=0", - "", # Wait for system to come up? + "e net.ifnames=0 biosdevnames=0 console=ttyS0", + "", # Wait for system to come up? "curl -o /tmp/install.sh http://{{ .HTTPIP }}:{{ .HTTPPort }}/install.sh", "", "/usr/bin/bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}" diff --git a/distros/archlinux/supports.yml b/distros/archlinux/supports.yml deleted file mode 100644 index 68b73d5..0000000 --- a/distros/archlinux/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -rolling: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/centos-stream/8/amd64.pkrvars.hcl.sh b/distros/centos-stream/8/amd64.pkrvars.hcl.sh-eol similarity index 97% rename from distros/centos-stream/8/amd64.pkrvars.hcl.sh rename to distros/centos-stream/8/amd64.pkrvars.hcl.sh-eol index 66bf967..915b18f 100755 --- a/distros/centos-stream/8/amd64.pkrvars.hcl.sh +++ b/distros/centos-stream/8/amd64.pkrvars.hcl.sh-eol @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -ex diff --git a/distros/centos-stream/9/amd64.pkrvars.hcl.sh b/distros/centos-stream/9/amd64.pkrvars.hcl.sh index 5efdb05..2e5c192 100755 --- a/distros/centos-stream/9/amd64.pkrvars.hcl.sh +++ b/distros/centos-stream/9/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -ex @@ -24,10 +24,11 @@ boot_command = [ "e", "", " inst.text inst.ks=cdrom:/ks.cfg", + " inst.notmux console=ttyS0", "x" ] cd_files = [ "distros/centos-stream/9/disc/*" ] vbox_os_type = "RedHat9_64" HERPDERP -rm -f cache/CentOS-Stream-9-latest-x86_64-boot.iso \ No newline at end of file +rm -f cache/CentOS-Stream-9-latest-x86_64-boot.iso diff --git a/distros/centos-stream/default.nix b/distros/centos-stream/default.nix new file mode 100644 index 0000000..a0837c6 --- /dev/null +++ b/distros/centos-stream/default.nix @@ -0,0 +1,9 @@ +[ { + version = "8"; + arch = "amd64"; + eol = true; +} { + version = "9"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/centos-stream/supports.yml b/distros/centos-stream/supports.yml deleted file mode 100644 index 533b6df..0000000 --- a/distros/centos-stream/supports.yml +++ /dev/null @@ -1,10 +0,0 @@ -8: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -9: *supp diff --git a/distros/centos/7/amd64.pkrvars.hcl b/distros/centos/7/amd64.pkrvars.hcl index b553bb1..1926391 100644 --- a/distros/centos/7/amd64.pkrvars.hcl +++ b/distros/centos/7/amd64.pkrvars.hcl @@ -8,6 +8,7 @@ boot_command = [ "e", "", " inst.ks=cdrom:/ks.cfg", + " inst.notmux console=ttyS0", "x" ] cd_files = ["distros/centos/7/disc/*"] diff --git a/distros/centos/default.nix b/distros/centos/default.nix new file mode 100644 index 0000000..061eca4 --- /dev/null +++ b/distros/centos/default.nix @@ -0,0 +1,5 @@ +[ { + version = "7"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/centos/supports.yml b/distros/centos/supports.yml deleted file mode 100644 index 6409bc4..0000000 --- a/distros/centos/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -7: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso \ No newline at end of file diff --git a/distros/debian/10/amd64.pkrvars.hcl b/distros/debian/10/amd64.pkrvars.hcl index 7a1aeb1..94d2556 100644 --- a/distros/debian/10/amd64.pkrvars.hcl +++ b/distros/debian/10/amd64.pkrvars.hcl @@ -16,6 +16,7 @@ boot_command = [ "debconf/frontend=noninteractive ", "hostname={{.Name}} ", "domain=vagrant ", + "console=ttyS0 ", "" ] http_directory = "distros/debian/10/http/" diff --git a/distros/debian/11/amd64.pkrvars.hcl b/distros/debian/11/amd64.pkrvars.hcl index 795b6cb..e93396e 100644 --- a/distros/debian/11/amd64.pkrvars.hcl +++ b/distros/debian/11/amd64.pkrvars.hcl @@ -16,6 +16,7 @@ boot_command = [ "debconf/frontend=noninteractive ", "hostname={{.Name}} ", "domain=vagrant ", + "console=ttyS0 ", "" ] http_directory = "distros/debian/11/http/" diff --git a/distros/debian/12/amd64.pkrvars.hcl b/distros/debian/12/amd64.pkrvars.hcl index d65e049..3278968 100644 --- a/distros/debian/12/amd64.pkrvars.hcl +++ b/distros/debian/12/amd64.pkrvars.hcl @@ -16,6 +16,7 @@ boot_command = [ "debconf/frontend=noninteractive ", "hostname={{.Name}} ", "domain=vagrant ", + "console=ttyS0 ", "" ] http_directory = "distros/debian/12/http/" diff --git a/distros/debian/default.nix b/distros/debian/default.nix new file mode 100644 index 0000000..f8a6e30 --- /dev/null +++ b/distros/debian/default.nix @@ -0,0 +1,20 @@ +[ { + version = "9"; + arch = "amd64"; + eol = true; +} { + version = "10"; + arch = "amd64"; + eol = false; +} { + version = "11"; + arch = "amd64"; + eol = false; +} { + version = "12"; + arch = "amd64"; + eol = false; + exclude = [ + "virtualbox-iso" + ]; +} ] diff --git a/distros/debian/supports.yml b/distros/debian/supports.yml deleted file mode 100644 index 9865444..0000000 --- a/distros/debian/supports.yml +++ /dev/null @@ -1,17 +0,0 @@ -10: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -11: *supp -12: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso \ No newline at end of file diff --git a/distros/default.nix b/distros/default.nix new file mode 100644 index 0000000..0fd1bf5 --- /dev/null +++ b/distros/default.nix @@ -0,0 +1,13 @@ +{ + almalinux = import ./almalinux; + alpine = import ./alpine; + archlinux = import ./archlinux; + centos = import ./centos; + centos-stream = import ./centos-stream; + debian = import ./debian; + fedora = import ./fedora; + nixos = import ./nixos; + opensuse = import ./opensuse; + rockylinux = import ./rockylinux; + ubuntu = import ./ubuntu; +} diff --git a/distros/fedora/39/amd64-ks.cfg b/distros/fedora/39/amd64-ks.cfg index 90ed9f8..ba67610 100644 --- a/distros/fedora/39/amd64-ks.cfg +++ b/distros/fedora/39/amd64-ks.cfg @@ -1,5 +1,5 @@ user --name=vagrant --password=vagrant -rootpw vagrant +rootpw --plaintext vagrant text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 lang en_US.UTF-8 @@ -7,7 +7,6 @@ keyboard us timezone --utc Etc/UTC selinux --enforcing -rootpw --lock --iscrypted locked firewall --disabled diff --git a/distros/fedora/39/amd64.pkrvars.hcl b/distros/fedora/39/amd64.pkrvars.hcl index 2a25ab1..6c3821c 100644 --- a/distros/fedora/39/amd64.pkrvars.hcl +++ b/distros/fedora/39/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e ", - "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0" + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0" ] cd_files = [ "distros/fedora/39/*.cfg" diff --git a/distros/fedora/40/amd64-ks.cfg b/distros/fedora/40/amd64-ks.cfg new file mode 100644 index 0000000..9e19b81 --- /dev/null +++ b/distros/fedora/40/amd64-ks.cfg @@ -0,0 +1,182 @@ +user --name=vagrant --password=vagrant +rootpw --plaintext vagrant + +text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 +lang en_US.UTF-8 +keyboard us +timezone --utc Etc/UTC + +selinux --enforcing + +firewall --disabled + +# We pass net.ifnames=0 because we always want to use eth0 here on all the cloud images. +bootloader --timeout=1 --location=mbr --append="no_timer_check net.ifnames=0 console=tty1 console=ttyS0,115200n8 biosdevnames=0" + +services --enabled=sshd + +# Configure for gpt with bios+uefi +clearpart --all --initlabel --disklabel=gpt +part prepboot --size=4 --fstype=prepboot +part biosboot --size=1 --fstype=biosboot +part /boot/efi --size=100 --fstype=efi +part /boot --size=1000 --fstype=ext4 --label=boot +part btrfs.007 --size=2000 --fstype=btrfs --grow +btrfs none --label=fedora btrfs.007 +btrfs /home --subvol --name=home LABEL=fedora +btrfs / --subvol --name=root LABEL=fedora + +repo --name=fedora --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch +repo --name=updates --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=updates-released-f$releasever&arch=$basearch +url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch + +reboot + +##### begin package list ############################################# +%packages --instLangs=en + +# Include packages for the cloud-server-environment group +@^cloud-server-environment + +# Install the tracer dnf plugin to enable automatic reboots +# IF the user requests package updates +# AND requests a reboot +# AND the packages updated require a reboot. +# https://fedoraproject.org/wiki/Changes/Automatic_Cloud_Reboot_On_Updates +python3-dnf-plugin-tracer + +# Don't include the kernel toplevel package since it pulls in +# kernel-modules. We're happy for now with kernel-core. +-kernel +kernel-core + +# Don't include dracut-config-rescue. It will have dracut generate a +# "rescue" entry in the grub menu, but that also means there is a +# rescue kernel and initramfs that get created, which (currently) add +# about another 40MiB to the /boot/ partition. Also the "rescue" mode +# is generally not useful in the cloud. +-dracut-config-rescue + +# Plymouth provides a graphical boot animation. In the cloud we don't +# need a graphical boot animation. This also means anaconda won't put +# rhgb/quiet on kernel command line +-plymouth + +# Install qemu-guest-agent https://pagure.io/cloud-sig/issue/319 To +# improve the integration with OpenStack and other VM management +# systems (oVirt, KubeVirt). +qemu-guest-agent + + +# No need for firewalld for now. We don't have a firewall on by default. +-firewalld + +# Don't include the geolite2 databases, which end up with 66MiB +# in /usr/share/GeoIP +-geolite2-country +-geolite2-city + +# Specific to vagrant +rsync +fuse-sshfs +%end +##### end package list ############################################### + + +##### begin kickstart post ########################################### +%post --erroronfail +#!/bin/bash +set -x +if [ -e /dev/vda ]; then + device="/dev/vda" +elif [ -e /dev/sda ]; then + device="/dev/sda" +else + echo "No device found" + exit 1 +fi + +virt="$(systemd-detect-virt)" +if [ "${virt}" == "microsoft" ]; then + dnf install -y hyperv-daemons + systemctl enable hpervfcopyd + systemctl enable hypervkvpd + systemctl enable hypervvssd +fi + +if [ "$(arch)" = "x86_64" ]; then +# Set up legacy BIOS boot if we booted from UEFI +grub2-install --target=i386-pc "${device}" +fi + +# Blivet sets pmbr_boot flag erroneously and we need to purge it +# otherwise it'll fail to boot +parted "${device}" disk_set pmbr_boot off + +# linux-firmware is installed by default and is quite large. As of mid 2020: +# Total download size: 97 M +# Installed size: 268 M +# So far we've been fine shipping without it so let's continue. +# More discussion about this in #1234504. +echo "Removing linux-firmware package." +rpm -e linux-firmware + +# See the systemd-random-seed.service man page that says: +# " It is recommended to remove the random seed from OS images intended +# for replication on multiple systems" +echo "Removing random-seed so it's not the same in every image." +rm -f /var/lib/systemd/random-seed + +echo "Import RPM GPG key" +releasever=$(rpm --eval '%{fedora}') +basearch=$(uname -i) +rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch + +echo "Zeroing out empty space." +# Create zeros file with nodatacow and no compression +touch /var/tmp/zeros +chattr +C /var/tmp/zeros +# This forces the filesystem to reclaim space from deleted files +dd bs=1M if=/dev/zero of=/var/tmp/zeros || : +echo "(Don't worry -- that out-of-space error was expected.)" +# Force sync to disk (Cf. https://pagure.io/cloud-sig/issue/340#comment-743430) +btrfs filesystem sync / +rm -f /var/tmp/zeros +btrfs filesystem sync / + +# When we build the image a networking config file gets left behind. +# Let's clean it up. +echo "Cleanup leftover networking configuration" +rm -f /etc/NetworkManager/system-connections/*.nmconnection +#*/ + +# Truncate the /etc/resolv.conf left over from NetworkManager during the +# kickstart. This causes delays in boot with cloud-init because the +# 192.168.122.1 DNS server cannot be reached. +truncate -s 0 /etc/resolv.conf + +# Clear machine-id on pre generated images +truncate -s 0 /etc/machine-id + +# Vagrant setup +sed -i 's,Defaults\\s*requiretty,Defaults !requiretty,' /etc/sudoers +echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant-nopasswd +sed -i 's/.*UseDNS.*/UseDNS no/' /etc/ssh/sshd_config + +cat > /etc/ssh/sshd_config.d/10-vagrant-insecure-rsa-key.conf <e ", + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0" +] +cd_files = [ + "distros/fedora/40/*.cfg" +] +vbox_os_type = "Fedora_64" diff --git a/distros/fedora/40/finalize.sh b/distros/fedora/40/finalize.sh new file mode 100644 index 0000000..c30e5a8 --- /dev/null +++ b/distros/fedora/40/finalize.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex + +dnf -y update + +dnf clean all diff --git a/distros/fedora/40/qemu.sh b/distros/fedora/40/qemu.sh new file mode 100644 index 0000000..569dc7c --- /dev/null +++ b/distros/fedora/40/qemu.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex + +dnf install -y qemu-guest-agent +systemctl start qemu-guest-agent +systemctl enable qemu-guest-agent diff --git a/distros/fedora/40/virtualbox.sh b/distros/fedora/40/virtualbox.sh new file mode 100644 index 0000000..209dab5 --- /dev/null +++ b/distros/fedora/40/virtualbox.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex + +dnf install -y virtualbox-guest-additions +systemctl start vboxservice +systemctl enable vboxservice diff --git a/distros/fedora/40/vmware.sh b/distros/fedora/40/vmware.sh new file mode 100644 index 0000000..058afc2 --- /dev/null +++ b/distros/fedora/40/vmware.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -ex + +dnf install -y open-vm-tools diff --git a/distros/fedora/default.nix b/distros/fedora/default.nix new file mode 100644 index 0000000..17da572 --- /dev/null +++ b/distros/fedora/default.nix @@ -0,0 +1,21 @@ +[ { + version = "37"; + arch = "amd64"; + eol = true; +} { + version = "38"; + arch = "amd64"; + eol = true; +} { + version = "39"; + arch = "amd64"; + eol = false; +} { + version = "40"; + arch = "amd64"; + eol = false; +} { + version = "rawhide"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/fedora/rawhide/amd64-ks.cfg b/distros/fedora/rawhide/amd64-ks.cfg index ccbb7d9..6d5fa77 100644 --- a/distros/fedora/rawhide/amd64-ks.cfg +++ b/distros/fedora/rawhide/amd64-ks.cfg @@ -1,5 +1,5 @@ user --name=vagrant --password=vagrant -rootpw vagrant +rootpw --plaintext vagrant text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 lang en_US.UTF-8 @@ -7,7 +7,6 @@ keyboard us timezone --utc Etc/UTC selinux --enforcing -rootpw --lock --iscrypted locked firewall --disabled @@ -77,7 +76,6 @@ qemu-guest-agent -geolite2-city # Specific to vagrant -dnf-yum rsync fuse-sshfs %end diff --git a/distros/fedora/rawhide/amd64.pkrvars.hcl.sh b/distros/fedora/rawhide/amd64.pkrvars.hcl.sh index dad3bd7..fa06d83 100755 --- a/distros/fedora/rawhide/amd64.pkrvars.hcl.sh +++ b/distros/fedora/rawhide/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -exo pipefail # Name of the file to write out @@ -35,7 +35,7 @@ iso = { } boot_command = [ "e", - "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0 console=ttyS0" + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0" ] cd_files = [ "distros/fedora/rawhide/*.cfg" @@ -44,4 +44,4 @@ vbox_os_type = "Fedora_64" EOF rm -r "${tmp}" -rm -f "cache/${iso_name}" \ No newline at end of file +rm -f "cache/${iso_name}" diff --git a/distros/fedora/rawhide/qemu.sh b/distros/fedora/rawhide/qemu.sh index 569dc7c..e44c536 100644 --- a/distros/fedora/rawhide/qemu.sh +++ b/distros/fedora/rawhide/qemu.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash set -ex -dnf install -y qemu-guest-agent systemctl start qemu-guest-agent systemctl enable qemu-guest-agent diff --git a/distros/fedora/supports.yml b/distros/fedora/supports.yml deleted file mode 100644 index 295db47..0000000 --- a/distros/fedora/supports.yml +++ /dev/null @@ -1,20 +0,0 @@ -37: # EOL -38: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -39: *supp -rawhide: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: hyperv-iso - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso diff --git a/distros/nixos/23.05/amd64.pkrvars.hcl.sh b/distros/nixos/23.05/amd64.pkrvars.hcl.sh index e46e09f..0dbb92a 100755 --- a/distros/nixos/23.05/amd64.pkrvars.hcl.sh +++ b/distros/nixos/23.05/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -exo pipefail # Name of the file to write out @@ -34,8 +34,8 @@ boot_command = [ "sudo bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}" ] http_directory = "distros/nixos/23.05/http/" -boot_wait = "60s" +boot_wait = "90s" EOF rm -r "${tmp}" -rm -f "cache/latest-nixos-minima-x86_64-linux.iso" \ No newline at end of file +rm -f "cache/latest-nixos-minima-x86_64-linux.iso" diff --git a/distros/nixos/23.11/amd64.pkrvars.hcl.sh b/distros/nixos/23.11/amd64.pkrvars.hcl.sh index 54e0d8f..1078713 100755 --- a/distros/nixos/23.11/amd64.pkrvars.hcl.sh +++ b/distros/nixos/23.11/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -exo pipefail # Name of the file to write out @@ -34,7 +34,7 @@ boot_command = [ "sudo bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}" ] http_directory = "distros/nixos/23.11/http/" -boot_wait = "60s" +boot_wait = "90s" EOF rm -r "${tmp}" diff --git a/distros/nixos/default.nix b/distros/nixos/default.nix new file mode 100644 index 0000000..f9e89b8 --- /dev/null +++ b/distros/nixos/default.nix @@ -0,0 +1,9 @@ +[ { + version = "23.05"; + arch = "amd64"; + eol = true; +} { + version = "23.11"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/nixos/supports.yml b/distros/nixos/supports.yml deleted file mode 100644 index d6edc34..0000000 --- a/distros/nixos/supports.yml +++ /dev/null @@ -1,10 +0,0 @@ -23.05: # EOL -23.11: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/opensuse/15.5/amd64.pkrvars.hcl b/distros/opensuse/15.5/amd64.pkrvars.hcl index 8b80ad3..ce2b718 100644 --- a/distros/opensuse/15.5/amd64.pkrvars.hcl +++ b/distros/opensuse/15.5/amd64.pkrvars.hcl @@ -9,6 +9,7 @@ boot_command = [ "", "", " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1", + " console=ttyS0", "x" ] http_directory = "distros/opensuse/15.5/http/" diff --git a/distros/opensuse/15.6/amd64.pkrvars.hcl b/distros/opensuse/15.6/amd64.pkrvars.hcl index 64a74be..8e158a1 100644 --- a/distros/opensuse/15.6/amd64.pkrvars.hcl +++ b/distros/opensuse/15.6/amd64.pkrvars.hcl @@ -9,6 +9,7 @@ boot_command = [ "", "", " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1", + " console=ttyS0", "x" ] http_directory = "distros/opensuse/15.6/http/" diff --git a/distros/opensuse/default.nix b/distros/opensuse/default.nix new file mode 100644 index 0000000..01de93d --- /dev/null +++ b/distros/opensuse/default.nix @@ -0,0 +1,9 @@ +[ { + version = "15.5"; + arch = "amd64"; + eol = false; +} { + version = "tumbleweed"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/opensuse/supports.yml b/distros/opensuse/supports.yml deleted file mode 100644 index 06d713d..0000000 --- a/distros/opensuse/supports.yml +++ /dev/null @@ -1,11 +0,0 @@ -15.5: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -15.6: -tumbleweed: *supp diff --git a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh index a9b1640..9a6fe9c 100755 --- a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh +++ b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh @@ -1,8 +1,8 @@ -#!/usr/bin/env bash +#!env bash name="${0%.sh}" -base="https://download.opensuse.org/tumbleweed/iso/openSUSE-Tumbleweed-NET-x86_64-Current.iso" +base="https://download.opensuse.org/tumbleweed/iso/openSUSE-Tumbleweed-DVD-x86_64-Current.iso" # Working directory tmp="$(mktemp -d)" @@ -26,10 +26,11 @@ boot_command = [ "e", "", " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1", + " console=ttyS0", "x" ] http_directory = "distros/opensuse/tumbleweed/http/" vbox_os_type = "OpenSUSE_Tumbleweed_64" EOF -rm -r "${tmp}" \ No newline at end of file +rm -r "${tmp}" diff --git a/distros/opensuse/tumbleweed/http/autoinst.xml b/distros/opensuse/tumbleweed/http/autoinst.xml index 68f4eec..774fa71 100644 --- a/distros/opensuse/tumbleweed/http/autoinst.xml +++ b/distros/opensuse/tumbleweed/http/autoinst.xml @@ -8,6 +8,7 @@ false true + true @@ -120,4 +121,4 @@ root ALL=(ALL) NOPASSWD: ALL https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant.pub - \ No newline at end of file + diff --git a/distros/rockylinux/9/amd64.pkrvars.hcl b/distros/rockylinux/9/amd64.pkrvars.hcl index fd49627..12da5e9 100644 --- a/distros/rockylinux/9/amd64.pkrvars.hcl +++ b/distros/rockylinux/9/amd64.pkrvars.hcl @@ -1,13 +1,14 @@ distro = "rockylinux" version = "9" iso = { - url = "http://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9.3-x86_64-boot.iso" - checksum = "eb096f0518e310f722d5ebd4c69f0322df4fc152c6189f93c5c797dc25f3d2e1" + url = "https://download.rockylinux.org/pub/rocky/9.4/isos/x86_64/Rocky-9.4-x86_64-boot.iso" + checksum = "c7e95e3dba88a1f68fff8b7d4e66adf6f76ac4fba2e246a83c46ab79574c78a8" } boot_command = [ "e", "", " inst.text inst.ks=cdrom:/ks.cfg", + " inst.notmux console=ttyS0", "x" ] cd_files = [ "distros/rockylinux/9/disc/*" ] diff --git a/distros/rockylinux/default.nix b/distros/rockylinux/default.nix new file mode 100644 index 0000000..a0837c6 --- /dev/null +++ b/distros/rockylinux/default.nix @@ -0,0 +1,9 @@ +[ { + version = "8"; + arch = "amd64"; + eol = true; +} { + version = "9"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/rockylinux/supports.yml b/distros/rockylinux/supports.yml deleted file mode 100644 index 533b6df..0000000 --- a/distros/rockylinux/supports.yml +++ /dev/null @@ -1,10 +0,0 @@ -8: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -9: *supp diff --git a/distros/ubuntu/20.04/amd64.pkrvars.hcl b/distros/ubuntu/20.04/amd64.pkrvars.hcl index 5c49cad..a95048e 100644 --- a/distros/ubuntu/20.04/amd64.pkrvars.hcl +++ b/distros/ubuntu/20.04/amd64.pkrvars.hcl @@ -7,7 +7,7 @@ iso = { boot_wait = "3s" boot_command = [ "e", - "console=tty0 ", + "console=ttyS0 ", "fsck.mode=skip ", "locale=en_US ", "auto=true ", diff --git a/distros/ubuntu/22.04/amd64.pkrvars.hcl b/distros/ubuntu/22.04/amd64.pkrvars.hcl index 96f325b..51baf9b 100644 --- a/distros/ubuntu/22.04/amd64.pkrvars.hcl +++ b/distros/ubuntu/22.04/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e", - "console=tty0 ", + "console=ttyS0 ", "fsck.mode=skip ", "locale=en_US ", "auto=true ", diff --git a/distros/ubuntu/23.10/amd64.pkrvars.hcl b/distros/ubuntu/23.10/amd64.pkrvars.hcl index 3c8832a..59d3759 100644 --- a/distros/ubuntu/23.10/amd64.pkrvars.hcl +++ b/distros/ubuntu/23.10/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e", - "console=tty0 ", + "console=ttyS0 ", "fsck.mode=skip ", "locale=en_US ", "auto=true ", diff --git a/distros/ubuntu/default.nix b/distros/ubuntu/default.nix new file mode 100644 index 0000000..c0643db --- /dev/null +++ b/distros/ubuntu/default.nix @@ -0,0 +1,17 @@ +[ { + version = "18.04"; + arch = "amd64"; + eol = true; +} { + version = "20.04"; + arch = "amd64"; + eol = false; +} { + version = "22.04"; + arch = "amd64"; + eol = false; +} { + version = "23.10"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/ubuntu/supports.yml b/distros/ubuntu/supports.yml deleted file mode 100644 index 80525fb..0000000 --- a/distros/ubuntu/supports.yml +++ /dev/null @@ -1,13 +0,0 @@ -20.04: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso - # linux-azure package broken in 20.04 -22.04: *supp -"23.10": *supp -#18.04: *supp # EOL diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..45463b6 --- /dev/null +++ b/flake.lock @@ -0,0 +1,194 @@ +{ + "nodes": { + "devshell": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1717408969, + "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=", + "owner": "numtide", + "repo": "devshell", + "rev": "1ebbe68d57457c8cae98145410b164b5477761f4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1717285511, + "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-root": { + "locked": { + "lastModified": 1713493429, + "narHash": "sha256-ztz8JQkI08tjKnsTpfLqzWoKFQF4JGu2LRz8bkdnYUk=", + "owner": "srid", + "repo": "flake-root", + "rev": "bc748b93b86ee76e2032eecda33440ceb2532fcd", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1704161960, + "narHash": "sha256-QGua89Pmq+FBAro8NriTuoO/wNaUtugt29/qqA8zeeM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63143ac2c9186be6d9da6035fa22620018c85932", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1717284937, + "narHash": "sha256-lIbdfCsf8LMFloheeE6N31+BMIeixqyQWbSr2vk79EQ=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1718870667, + "narHash": "sha256-jab3Kpc8O1z3qxwVsCMHL4+18n5Wy/HHKyu1fcsF7gs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9b10b8f00cb5494795e5f51b39210fed4d2b0748", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1708475490, + "narHash": "sha256-g1v0TsWBQPX97ziznfJdWhgMyMGtoBFs102xSYO4syU=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "0e74ca98a74bc7270d28838369593635a5db3260", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "process-compose": { + "locked": { + "lastModified": 1718031437, + "narHash": "sha256-+RrlkAVZx0QhyeHAGFJnjST+/7Dc3zsDU3zAKXoDXaI=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "9344fac44edced4c686721686a6ad904d067c546", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, + "root": { + "inputs": { + "devshell": "devshell", + "flake-parts": "flake-parts", + "flake-root": "flake-root", + "nixpkgs": "nixpkgs_2", + "process-compose": "process-compose", + "treefmt-nix": "treefmt-nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1718522839, + "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..0febf82 --- /dev/null +++ b/flake.nix @@ -0,0 +1,193 @@ +# github.com/brianmcgee has lots of great ideas for going above and beyond for this +{ + description = "Greg's homepage"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + devshell.url = "github:numtide/devshell"; + flake-parts.url = "github:hercules-ci/flake-parts"; + flake-root.url = "github:srid/flake-root"; + process-compose.url = "github:Platonic-Systems/process-compose-flake"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + }; + + outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, nixpkgs, ... }@inputs: let + lib = nixpkgs.lib; + providers = [ + "qemu" + "virtualbox-iso" + "hyperv-iso" + ]; + distros = import ./distros; + # Inserts the distro name into each element in the list + insertNames = lib.attrsets.mapAttrsToList (distro: val: (builtins.map (x: x // { inherit distro; }) val)) distros; + # Flattens all the distros into a single list + flat = lib.lists.flatten insertNames; + # Combines providers list with flattened list + withProviders = lib.attrsets.mapCartesianProduct + ({d, provider}: d // { inherit provider; }) + { d = flat; provider = providers; }; + active = builtins.filter (x: (x?eol) && (!x.eol)) withProviders; + name = d: "-${d.distro}-${d.arch}-${d.provider}-${d.version}"; + distroFile = d: "distros/${d.distro}/${d.version}/${d.arch}.pkrvars.hcl"; + cmd = d: '' + version="''${1:-1}" + packer build -var "build=$version" -only=${d.provider}.${d.arch} -var-file=${distroFile d}''; + in flake-parts.lib.mkFlake { inherit inputs; } + { + imports = [ + inputs.devshell.flakeModule + inputs.flake-root.flakeModule + inputs.treefmt-nix.flakeModule + ]; + flake = { + # Universal things + }; + systems = [ + "x86_64-linux" + "aarch64-linux" + ]; + perSystem = { config, pkgs, self', system, ... }: + { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + config.allowUnfree = true; + }; + # https://numtide.github.io/devshell/intro.html + devshells.default = { + # These commands will be shown in the motd that users read + # upon entering the shell + commands = [ { + package = self'.packages.cache; + category = "utils"; + help = "Downloads ISO files and adds them to the S3 bucket"; + } { + package = self'.packages.generate; + category = "utils"; + help = "Generate dynamic HCL files"; + } { + package = self'.packages.shellcheck; + category = "utils"; + help = "Run shellchecks"; + } { + package = self'.packages.validate; + category = "utils"; + help = "Validate all the files"; + } { + package = pkgs.vagrant; + category = "utils"; + help = "Vagrant to help with testing"; + } ] ++ (builtins.map (d: { package = self'.packages."build${name d}"; category = "builds"; }) withProviders); + env = [{ + name = "EFI_DIR"; + value = "${pkgs.OVMF.fd}/FV/"; + }]; + }; + +# This is not special, this is just flakes + packages = let + base = { + runtimeInputs = with pkgs; [ + packer + qemu_full + vagrant + ]; + }; + build = d: let + vp = x: if x.provider == "qemu" then "libvirt" + else if x.provider == "virtualbox-iso" then "virtualbox" + else if x.provider == "hyperv-iso" then "hyperv" + else "unknown"; + in pkgs.writeShellApplication (base // { + name = "build${name d}"; + text = '' + set -x + ${cmd d} -except=upload -var headless=false sources/ + vagrant box add --provider ${vp d} ./output/${vp d}/${d.distro}-${d.version}.box --name test_box --force + ''; + }); + upload = d: pkgs.writeShellApplication (base // { + name = "upload${name d}"; + text = '' + set -x + ${cmd d} sources/ + ''; + }); + builds = lib.lists.foldr (d: acc: acc // { "build${name d}" = (build d); }) {} withProviders; + uploads = lib.lists.foldr (d: acc: acc // { "upload${name d}" = (upload d); }) {} withProviders; + qemu_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "qemu") active)); + vbox_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "virtualbox-iso") active)); + hyperv_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "hyperv-iso") active)); + in builds // uploads // { + continue = pkgs.writeText "continue.yml" + (builtins.replaceStrings ["QEMU_MATRIX" "VBOX_MATRIX" "HYPERV_MATRIX"] [qemu_list vbox_list hyperv_list] + (builtins.readFile ./.gitlab-ci-build.yml.in) + ); + cache = pkgs.callPackage (import ./nix/cache.nix) { inherit active distroFile; }; + cache-restore = pkgs.writeShellApplication { + name = "cache-restore"; + runtimeInputs = with pkgs; [ + jq + minio-client + packer + ]; + text = builtins.readFile ./cache-restore.sh; + }; + generate = pkgs.writeShellApplication { + name = "generate"; + runtimeInputs = with pkgs; [ + bash + curl + jq + pup + ]; + text = '' + shopt -s globstar + for vars in distros/**/*.pkrvars.hcl.sh; do + bash "./''${vars}" + done + ''; + }; + shellcheck = pkgs.writeShellApplication { + name = "shellcheck"; + runtimeInputs = with pkgs; [ + shellcheck + ]; + text = '' + scripts=() + shopt -s globstar + for f in **/*.sh; do + scripts+=("$f") + done + shellcheck -e SC2239 "''${scripts[@]}" + ''; + }; + validate = pkgs.writeShellApplication { + name = "validate"; + runtimeInputs = with pkgs; [ + packer + ]; + text = '' + shopt -s globstar + packer init sources + for vars in distros/**/*.pkrvars.hcl; do + packer validate -except=upload "-var-file=''${vars}" sources/ + done + ''; + }; + }; + + # https://numtide.github.io/treefmt/ + treefmt.config = { + inherit (config.flake-root) projectRootFile; + package = pkgs.treefmt; + + programs = { + deadnix.enable = true; + }; + + settings.formatter = { + }; + }; + }; + }; +} diff --git a/lib/__init__.py b/lib/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/build.py b/lib/build.py deleted file mode 100644 index a0fa1c2..0000000 --- a/lib/build.py +++ /dev/null @@ -1,40 +0,0 @@ -from provider import Provider - - -class Build: - def __init__(self, distro, version, provider: Provider): - self.distro: str = distro - self.version: str = str(version) - self.provider: Provider = provider - - @property - def var_file(self): - return f"-var-file={self.var_path}" - - @property - def matrix(self): - return { - "builder": str(self.provider), - "distro": self.var_path, - "runner": self.provider.runner, - } - - @property - def is_github(self) -> bool: - if self.provider.runner is not None: - return True - return False - - @property - def var_path(self): - return f"distros/{self.distro}/{self.version}/{self.provider.arch}.pkrvars.hcl" - - @property - def only(self): - return f"-only={self.provider}" - - def __str__(self) -> str: - return f"{self.distro} {self.version}: {self.provider}" - - def __repr__(self) -> str: - return str(self) diff --git a/lib/distro.py b/lib/distro.py deleted file mode 100644 index bd3bdfa..0000000 --- a/lib/distro.py +++ /dev/null @@ -1,28 +0,0 @@ -from provider import Provider -from build import Build - - -class Distro: - def __init__(self, name: str, support: map): - self.name = name - self.builds = [] - for version, matrix in support.items(): - if not matrix: - continue - for support in matrix: - provider = Provider(support["provider"], support["arch"]) - build = Build(self.name, version, provider) - self.builds.append(build) - - def __str__(self) -> str: - return f"{self.name}" - - def __repr__(self) -> str: - return f"{self.name}: {' '.join(self.versions)}" - - @property - def versions(self) -> set[str]: - return set([b.version for b in self.builds]) - - def builds_for_version(self, version: str) -> set[Build]: - return set(filter(lambda x: x.version == version, self.builds)) diff --git a/lib/provider.py b/lib/provider.py deleted file mode 100644 index 60f2bd4..0000000 --- a/lib/provider.py +++ /dev/null @@ -1,17 +0,0 @@ -class Provider: - def __init__(self, name, arch): - self.name = name - self.arch = arch - - def __str__(self) -> str: - return f"{self.name}.{self.arch}" - - @property - def runner(self) -> str: - if self.name == "qemu": - return "qemu" - elif self.name.startswith("virtualbox"): - return "virtualbox" - elif self.name.startswith("hyperv"): - return "hyperv" - return None diff --git a/lib/support.py b/lib/support.py deleted file mode 100644 index 26571cb..0000000 --- a/lib/support.py +++ /dev/null @@ -1,23 +0,0 @@ -from distro import Distro -from provider import Provider -from pathlib import Path -from yaml import load, Loader - - -BASE = Path(__file__).parent.parent.resolve() - - -class Support: - def __init__(self): - self.base = BASE - self.distros = self.__get_distros() - - def __get_distros(self) -> map: - distros = {} - distros_dir = self.base / "distros" - for f in distros_dir.glob("**/supports.yml"): - name = f.parent.name - with open(f, "r") as fp: - support_yml = load(fp, Loader=Loader) - distros[name] = Distro(name, support_yml) - return distros diff --git a/nix/cache.nix b/nix/cache.nix new file mode 100644 index 0000000..daadc80 --- /dev/null +++ b/nix/cache.nix @@ -0,0 +1,43 @@ +{ + writeShellApplication, + active, + distroFile, + + curl, + minio-client, + packer, + + ... +}: + +let + inherit (builtins) concatStringsSep map; +in writeShellApplication { + name = "cache-isos"; + runtimeInputs = [ + curl + minio-client + packer + ]; + text = '' + push="''${1:-yes}" + if [ "$push" == "yes" ]; then + mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + fi + mkdir -p cache + '' + (concatStringsSep "\n" ( + map (d: '' + url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) + sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) + # shellcheck disable=SC2143 + if [ -z "$(mc ls storage/isos/cache/ | grep "$sha")" ]; then + curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" + if [ "$push" == "yes" ]; then + mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso" + fi + else + echo "Skipping ${d.distro} - cache/$sha.iso" + fi + '') active + )); +} diff --git a/sources/hyperv-iso.pkr.hcl b/sources/hyperv-iso.pkr.hcl index 614245f..0ce8ff9 100644 --- a/sources/hyperv-iso.pkr.hcl +++ b/sources/hyperv-iso.pkr.hcl @@ -1,6 +1,6 @@ source "hyperv-iso" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}", var.iso.url] - iso_target_path = "cache/${basename(var.iso.url)}" + iso_urls = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_target_path = "cache/${var.iso.checksum}.iso" iso_checksum = var.iso.checksum output_directory = "output/hyperv/${local.name}" diff --git a/sources/inputs.pkr.hcl b/sources/inputs.pkr.hcl index 711ff8e..7234a1d 100644 --- a/sources/inputs.pkr.hcl +++ b/sources/inputs.pkr.hcl @@ -109,6 +109,16 @@ variable "vbox_os_type" { default = "Linux_64" } +variable "storage_url" { + type = string + default = env("STORAGE_URL") +} + +variable "minio_secret" { + type = string + default = env("MINIO_SECRET") +} + locals { name = "${var.distro}-${var.version}" description = templatefile("../README.box.md", { diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index bb3ecba..d1b82ee 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -4,8 +4,8 @@ variable "qemu_accelerator" { } source "qemu" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}", var.iso.url] - iso_target_path = "cache/${basename(var.iso.url)}" + iso_urls = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_target_path = "cache/${var.iso.checksum}.iso" iso_checksum = var.iso.checksum output_directory = "output/libvirt/${local.name}" @@ -35,6 +35,7 @@ source "qemu" "amd64" { ["-chardev", "socket,path=${local.name}.qga.sock,server=on,wait=off,id=qga0"], ["-device", "virtio-serial"], ["-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"], + ["-serial", "stdio"] ] accelerator = var.qemu_accelerator diff --git a/sources/virtualbox-iso.pkr.hcl b/sources/virtualbox-iso.pkr.hcl index 452e97a..e2c9a56 100644 --- a/sources/virtualbox-iso.pkr.hcl +++ b/sources/virtualbox-iso.pkr.hcl @@ -1,6 +1,6 @@ source "virtualbox-iso" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}", var.iso.url] - iso_target_path = "cache/${basename(var.iso.url)}" + iso_urls = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_target_path = "cache/${var.iso.checksum}.iso" iso_checksum = var.iso.checksum output_directory = "output/virtualbox/${local.name}" diff --git a/test.xsh b/test.xsh deleted file mode 100755 index 7259672..0000000 --- a/test.xsh +++ /dev/null @@ -1,88 +0,0 @@ -#!env xonsh -# vim: set ft=xonsh: -import argparse -import pathlib -import re -import sys -import time -from shutil import rmtree - - -def get_vagrant_provider(provider): - "Vagrant provider is not always the same as Packer provider" - if provider.startswith("qemu"): - return "libvirt" - elif provider.startswith("virtualbox"): - return "virtualbox" - elif provider.startswith("vmware"): - return "vmware" - elif provider.startswith("hyperv"): - return "hyperv" - else: - return provider - - -def get_providers(): - "List of providers as defined in sources/" - providers: set[str] = { - "qemu.amd64" - "virtualbox-iso.amd64", - "vmware-iso.amd64" - } - return providers - - -def get_parser(args, unsupported=set()): - parser = argparse.ArgumentParser("Builder") - parser.add_argument("--headless", action="store_true") - parser.add_argument("--upload", action="store_true") - parser.add_argument("--provider", action="append") - parsed = parser.parse_args(args) - opts = {} - if parsed.headless: - opts["headless"] = "headless=true" - else: - opts["headless"] = "headless=false" - if parsed.upload: - opts["upload"] = "" - else: - opts["upload"] = "-except=upload" - if parsed.provider: - opts["providers"] = set(parsed.provider) - else: - opts["providers"] = get_providers() - # Filter providers - opts["providers"] -= unsupported - return opts - - -def main(): - # Equivalent of `set -e` in Bash - $RAISE_SUBPROC_ERROR = True - if len(sys.argv) == 2: - dry_run = True - else: - dry_run = False - for box in g`output/**/*.box`: - p = pathlib.Path(box) - with open("Vagrantfile.in", "r") as fr: - content = fr.read() - content = content.replace("@@BOX@@", p.resolve().as_uri()) - content = content.replace("@@BOX_NAME@@", p.name) - with open("Vagrantfile", "w") as fw: - fw.write(content) - if not dry_run: - if str(p.parent.name) in ["virtualbox", "libvirt", "vmware", "hyperv"]: - vagrant --provider @(p.parent.name) box add @(box) --name @(p.name) --force - vagrant up --provider @(p.parent.name) - else: - print(f"Now whatcha wanna go and do that for? {box}") - continue - vagrant destroy -f - vag_folder = pathlib.Path(".vagrant") - if vag_folder.exists(): - rmtree(vag_folder) - - -if __name__ == '__main__': - main()