From 146be6b12908fac96c521b92fa7eeba970557d05 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 12:25:03 -0500 Subject: [PATCH 01/53] Use new image by default --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9bbd705..a50136b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,9 @@ variables: PACKER_CONFIG_DIR: ./packer_config/ PACKER_CACHE_DIR: ./cache/ +default: + image: $CI_REGISTRY_DIRECT/greg/ci-images/vm-test:latest + generate: stage: generate cache: @@ -24,7 +27,6 @@ generate: distros: stage: generate - image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest artifacts: reports: dotenv: build.env @@ -64,7 +66,6 @@ shellcheck: # could get to them downloads: stage: cache - image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest cache: - key: isofiles paths: From 20286d069776c6187d1e998c75fb9c7121cb28fc Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 12:31:30 -0500 Subject: [PATCH 02/53] Try different path --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a50136b..97008c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: PACKER_CACHE_DIR: ./cache/ default: - image: $CI_REGISTRY_DIRECT/greg/ci-images/vm-test:latest + image: $CI_REGISTRY/greg/ci-images/vm-test:latest generate: stage: generate From 669068a902c3551666ebc4479475e33046e3de11 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 12:42:55 -0500 Subject: [PATCH 03/53] Skip the port number --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97008c7..e17b8b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: PACKER_CACHE_DIR: ./cache/ default: - image: $CI_REGISTRY/greg/ci-images/vm-test:latest + image: registry.thehellings.com/greg/ci-images/vm-test:latest generate: stage: generate From 822e1c869d80035dc0750537fbf8b45db3729005 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 14:21:00 -0500 Subject: [PATCH 04/53] Do not hardcode env --- distros/centos-stream/8/amd64.pkrvars.hcl.sh | 2 +- distros/centos-stream/9/amd64.pkrvars.hcl.sh | 4 ++-- distros/fedora/rawhide/amd64.pkrvars.hcl.sh | 4 ++-- distros/nixos/23.05/amd64.pkrvars.hcl.sh | 4 ++-- distros/nixos/23.11/amd64.pkrvars.hcl.sh | 2 +- distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/distros/centos-stream/8/amd64.pkrvars.hcl.sh b/distros/centos-stream/8/amd64.pkrvars.hcl.sh index 66bf967..915b18f 100755 --- a/distros/centos-stream/8/amd64.pkrvars.hcl.sh +++ b/distros/centos-stream/8/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -ex diff --git a/distros/centos-stream/9/amd64.pkrvars.hcl.sh b/distros/centos-stream/9/amd64.pkrvars.hcl.sh index 5efdb05..f0fa813 100755 --- a/distros/centos-stream/9/amd64.pkrvars.hcl.sh +++ b/distros/centos-stream/9/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -ex @@ -30,4 +30,4 @@ cd_files = [ "distros/centos-stream/9/disc/*" ] vbox_os_type = "RedHat9_64" HERPDERP -rm -f cache/CentOS-Stream-9-latest-x86_64-boot.iso \ No newline at end of file +rm -f cache/CentOS-Stream-9-latest-x86_64-boot.iso diff --git a/distros/fedora/rawhide/amd64.pkrvars.hcl.sh b/distros/fedora/rawhide/amd64.pkrvars.hcl.sh index dad3bd7..fccdaab 100755 --- a/distros/fedora/rawhide/amd64.pkrvars.hcl.sh +++ b/distros/fedora/rawhide/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -exo pipefail # Name of the file to write out @@ -44,4 +44,4 @@ vbox_os_type = "Fedora_64" EOF rm -r "${tmp}" -rm -f "cache/${iso_name}" \ No newline at end of file +rm -f "cache/${iso_name}" diff --git a/distros/nixos/23.05/amd64.pkrvars.hcl.sh b/distros/nixos/23.05/amd64.pkrvars.hcl.sh index e46e09f..b74a8da 100755 --- a/distros/nixos/23.05/amd64.pkrvars.hcl.sh +++ b/distros/nixos/23.05/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -exo pipefail # Name of the file to write out @@ -38,4 +38,4 @@ boot_wait = "60s" EOF rm -r "${tmp}" -rm -f "cache/latest-nixos-minima-x86_64-linux.iso" \ No newline at end of file +rm -f "cache/latest-nixos-minima-x86_64-linux.iso" diff --git a/distros/nixos/23.11/amd64.pkrvars.hcl.sh b/distros/nixos/23.11/amd64.pkrvars.hcl.sh index 54e0d8f..73a5861 100755 --- a/distros/nixos/23.11/amd64.pkrvars.hcl.sh +++ b/distros/nixos/23.11/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash set -exo pipefail # Name of the file to write out diff --git a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh index a9b1640..7193d68 100755 --- a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh +++ b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!env bash name="${0%.sh}" @@ -32,4 +32,4 @@ http_directory = "distros/opensuse/tumbleweed/http/" vbox_os_type = "OpenSUSE_Tumbleweed_64" EOF -rm -r "${tmp}" \ No newline at end of file +rm -r "${tmp}" From cc08a440ff961de82ea9d63e3ce10a1ed5836178 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 14:49:39 -0500 Subject: [PATCH 05/53] Explicitly call bash --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e17b8b8..948519a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,7 @@ generate: - distros/**/*.pkrvars.hcl script: |- for vars in $(find distros/ -name '*.pkrvars.hcl.sh'); do - "./${vars}" + bash "./${vars}" done distros: From 087a1ee58c558da45dc52bd690ab9b4d1c6b42df Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 15:55:49 -0500 Subject: [PATCH 06/53] Ignore demanded absolute paths --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 948519a..a1650f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ shellcheck: stage: lint script: |- shellcheck --version - shellcheck $(find . -name '*.sh') + shellcheck -e SC2239 $(find . -name '*.sh') # Downloads all the ISO files so that we have a single cache with all # of them present. This prevents a race condition where some of the From 2baf005b84e1042cd34087d17318d700849fd46f Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 16 Apr 2024 23:02:23 -0500 Subject: [PATCH 07/53] Tell builders to skip HTTP --- sources/hyperv-iso.pkr.hcl | 2 +- sources/qemu.pkr.hcl | 2 +- sources/virtualbox-iso.pkr.hcl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sources/hyperv-iso.pkr.hcl b/sources/hyperv-iso.pkr.hcl index 614245f..f3f3adf 100644 --- a/sources/hyperv-iso.pkr.hcl +++ b/sources/hyperv-iso.pkr.hcl @@ -1,5 +1,5 @@ source "hyperv-iso" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}", var.iso.url] + iso_urls = [ "cache/${basename(var.iso.url)}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/hyperv/${local.name}" diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index bb3ecba..0cc7e2c 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -4,7 +4,7 @@ variable "qemu_accelerator" { } source "qemu" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}", var.iso.url] + iso_urls = [ "cache/${basename(var.iso.url)}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/libvirt/${local.name}" diff --git a/sources/virtualbox-iso.pkr.hcl b/sources/virtualbox-iso.pkr.hcl index 452e97a..37b2c74 100644 --- a/sources/virtualbox-iso.pkr.hcl +++ b/sources/virtualbox-iso.pkr.hcl @@ -1,5 +1,5 @@ source "virtualbox-iso" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}", var.iso.url] + iso_urls = [ "cache/${basename(var.iso.url)}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/virtualbox/${local.name}" From a5469d10737d6fbfad90f9dce16931f1047e463c Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 17 Apr 2024 15:51:25 -0500 Subject: [PATCH 08/53] Try using ISOs as artifacts instead of cache --- .gitlab-ci-build.yml.in | 6 +++--- .gitlab-ci.yml | 10 +++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 6da64c2..51f8aa9 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -11,9 +11,6 @@ stages: - sources/build.pkr.hcl paths: - ./packer_config/ - - key: isofiles - paths: - - cache/*.iso rules: - if: $CI_COMMIT_TAG variables: @@ -28,6 +25,9 @@ stages: - pipeline: $PARENT_PIPELINE_ID job: generate artifacts: true + - pipeline: $PARENT_PIPELINE_ID + job: download + artifacts: true tags: - ${tag} variables: &variables diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1650f0..723897c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,10 +13,6 @@ default: generate: stage: generate - cache: - - key: isofiles - paths: - - cache/*.iso artifacts: paths: - distros/**/*.pkrvars.hcl @@ -66,10 +62,10 @@ shellcheck: # could get to them downloads: stage: cache + artifacts: + paths: + - cache/*.iso cache: - - key: isofiles - paths: - - cache/*.iso - key: files: - sources/build.pkr.hcl From f20c5c714ea32a8b0ab97636ec4b1113216ab46b Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 20 Apr 2024 02:45:12 -0500 Subject: [PATCH 09/53] Auto-retry the downloads --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 723897c..e4aa321 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,6 +62,7 @@ shellcheck: # could get to them downloads: stage: cache + retry: 3 artifacts: paths: - cache/*.iso From 2bd1d75d72c658ceb0b7d3dbc3d50f5d20e6e5d9 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 20 Apr 2024 02:46:25 -0500 Subject: [PATCH 10/53] Apparently, you can only retry twice --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4aa321..43fa7d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,7 @@ shellcheck: # could get to them downloads: stage: cache - retry: 3 + retry: 2 artifacts: paths: - cache/*.iso From 3e360a000a6100da9f96f3e6d5e69ee39e01cfba Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 20 Apr 2024 15:11:21 +0000 Subject: [PATCH 11/53] Tell curl to also retry --- cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache.sh b/cache.sh index ab406a0..dccc27a 100755 --- a/cache.sh +++ b/cache.sh @@ -10,7 +10,7 @@ for vars in $(find distros/ -name '*.pkrvars.hcl'); do if [ -e "cache/${base}" ]; then echo "Found cache/${base} already" else - files+=("-O" "${file}") + files+=("--retry" "3" "--retry-all-errors" "-O" "${file}") fi done From 160b065cac8d11ab4dc86375bd519a37d49f4423 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sun, 21 Apr 2024 16:32:15 +0000 Subject: [PATCH 12/53] Try setting cache policy --- .gitlab-ci-build.yml.in | 7 ++++--- .gitlab-ci.yml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 51f8aa9..6a69f53 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -11,6 +11,10 @@ stages: - sources/build.pkr.hcl paths: - ./packer_config/ + - key: isofiles + paths: + - ./cache/*.iso + policy: pull rules: - if: $CI_COMMIT_TAG variables: @@ -25,9 +29,6 @@ stages: - pipeline: $PARENT_PIPELINE_ID job: generate artifacts: true - - pipeline: $PARENT_PIPELINE_ID - job: download - artifacts: true tags: - ${tag} variables: &variables diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43fa7d0..df91067 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,15 +63,16 @@ shellcheck: downloads: stage: cache retry: 2 - artifacts: - paths: - - cache/*.iso cache: - key: files: - sources/build.pkr.hcl paths: - ./packer_config/ + - key: isofiles + paths: + - ./cache/*.iso + policy: push-pull script: - packer init sources - bash cache.sh From 00a41fd7e2fd67a1d160056df4a41a0c7e60400e Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sun, 21 Apr 2024 16:35:27 +0000 Subject: [PATCH 13/53] Fish cache policy --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df91067..930f921 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,7 @@ downloads: - key: isofiles paths: - ./cache/*.iso - policy: push-pull + policy: pull-push script: - packer init sources - bash cache.sh From 6cadd7193b485ad3d1f09575b30a6e6dfa7c3b86 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sun, 21 Apr 2024 14:53:31 -0500 Subject: [PATCH 14/53] Output file list before run --- .gitlab-ci-build.yml.in | 1 + cache.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 6a69f53..13bef34 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -47,6 +47,7 @@ stages: - packer init sources - echo $COMMAND script: + - ls -l cache/ - echo $COMMAND > build.sh - bash build.sh || exit 1 diff --git a/cache.sh b/cache.sh index dccc27a..91e45dc 100755 --- a/cache.sh +++ b/cache.sh @@ -19,3 +19,5 @@ pushd cache echo "Downloading ${files[*]}" curl --parallel -L "${files[@]}" popd + +ls -l cache/ From 12e94263ef858b2463d1d1612c74fe88337641ea Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sun, 21 Apr 2024 15:07:50 -0500 Subject: [PATCH 15/53] Update cache properly --- cache.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cache.sh b/cache.sh index 91e45dc..f7fad98 100755 --- a/cache.sh +++ b/cache.sh @@ -3,6 +3,9 @@ set -eo pipefail files=() +# These files are always out of date +rm -f cache/*latest* cache/*Rawhide* + # shellcheck disable=SC2044 for vars in $(find distros/ -name '*.pkrvars.hcl'); do file=$(echo var.iso.url | packer console -var-file="${vars}" -config-type=hcl2 sources/) From e976908b3f092908e049cb82a6c9c8b0826f0bfd Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Thu, 20 Jun 2024 22:16:26 -0500 Subject: [PATCH 16/53] Remove CentOS Stream 8 Fixes #1 --- distros/centos-stream/supports.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/distros/centos-stream/supports.yml b/distros/centos-stream/supports.yml index 533b6df..175fab4 100644 --- a/distros/centos-stream/supports.yml +++ b/distros/centos-stream/supports.yml @@ -1,4 +1,4 @@ -8: &supp +9: &supp - arch: amd64 provider: qemu - arch: amd64 @@ -7,4 +7,3 @@ provider: vmware-iso - arch: amd64 provider: hyperv-iso -9: *supp From b5ffbac6643201e067e2ad85c4de8316995527d1 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Thu, 20 Jun 2024 22:20:28 -0500 Subject: [PATCH 17/53] Prevent EOL from moving --- .../8/{amd64.pkrvars.hcl.sh => amd64.pkrvars.hcl.sh-eol} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename distros/centos-stream/8/{amd64.pkrvars.hcl.sh => amd64.pkrvars.hcl.sh-eol} (100%) diff --git a/distros/centos-stream/8/amd64.pkrvars.hcl.sh b/distros/centos-stream/8/amd64.pkrvars.hcl.sh-eol similarity index 100% rename from distros/centos-stream/8/amd64.pkrvars.hcl.sh rename to distros/centos-stream/8/amd64.pkrvars.hcl.sh-eol From d16e79b8505632e381a5d192c5b509b690a8cd27 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 00:05:57 -0500 Subject: [PATCH 18/53] Version updates Retire Alma and Rocky 8 Update Alma/Rocky 9 to 9.4 Retire Fedora 38 Add Fedora 40 --- distros/almalinux/9/amd64.pkrvars.hcl | 4 +- distros/almalinux/supports.yml | 3 +- distros/fedora/40/amd64-ks.cfg | 184 +++++++++++++++++++++++++ distros/fedora/40/amd64.pkrvars.hcl | 14 ++ distros/fedora/40/finalize.sh | 6 + distros/fedora/40/qemu.sh | 6 + distros/fedora/40/virtualbox.sh | 6 + distros/fedora/40/vmware.sh | 4 + distros/fedora/supports.yml | 5 +- distros/rockylinux/9/amd64.pkrvars.hcl | 4 +- distros/rockylinux/supports.yml | 3 +- 11 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 distros/fedora/40/amd64-ks.cfg create mode 100644 distros/fedora/40/amd64.pkrvars.hcl create mode 100644 distros/fedora/40/finalize.sh create mode 100644 distros/fedora/40/qemu.sh create mode 100644 distros/fedora/40/virtualbox.sh create mode 100644 distros/fedora/40/vmware.sh diff --git a/distros/almalinux/9/amd64.pkrvars.hcl b/distros/almalinux/9/amd64.pkrvars.hcl index 0aaffa1..83de82e 100644 --- a/distros/almalinux/9/amd64.pkrvars.hcl +++ b/distros/almalinux/9/amd64.pkrvars.hcl @@ -1,8 +1,8 @@ distro = "almalinux" version = "9" iso = { - url = "https://mirrors.kernel.org/almalinux/9.3/isos/x86_64/AlmaLinux-9.3-x86_64-boot.iso" - checksum = "af5377a1d16bbe599ea91a8761ad645f2f54687075802bdc0c0703ee610182e9" + url = "https://mirrors.kernel.org/almalinux/9.4/isos/x86_64/AlmaLinux-9.4-x86_64-boot.iso" + checksum = "1e5d7da3d84d5d9a5a1177858a5df21b868390bfccf7f0f419b1e59acc293160" } boot_command = [ "e", diff --git a/distros/almalinux/supports.yml b/distros/almalinux/supports.yml index 533b6df..175fab4 100644 --- a/distros/almalinux/supports.yml +++ b/distros/almalinux/supports.yml @@ -1,4 +1,4 @@ -8: &supp +9: &supp - arch: amd64 provider: qemu - arch: amd64 @@ -7,4 +7,3 @@ provider: vmware-iso - arch: amd64 provider: hyperv-iso -9: *supp diff --git a/distros/fedora/40/amd64-ks.cfg b/distros/fedora/40/amd64-ks.cfg new file mode 100644 index 0000000..90ed9f8 --- /dev/null +++ b/distros/fedora/40/amd64-ks.cfg @@ -0,0 +1,184 @@ +user --name=vagrant --password=vagrant +rootpw vagrant + +text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 +lang en_US.UTF-8 +keyboard us +timezone --utc Etc/UTC + +selinux --enforcing +rootpw --lock --iscrypted locked + +firewall --disabled + +# We pass net.ifnames=0 because we always want to use eth0 here on all the cloud images. +bootloader --timeout=1 --location=mbr --append="no_timer_check net.ifnames=0 console=tty1 console=ttyS0,115200n8 biosdevnames=0" + +services --enabled=sshd + +# Configure for gpt with bios+uefi +clearpart --all --initlabel --disklabel=gpt +part prepboot --size=4 --fstype=prepboot +part biosboot --size=1 --fstype=biosboot +part /boot/efi --size=100 --fstype=efi +part /boot --size=1000 --fstype=ext4 --label=boot +part btrfs.007 --size=2000 --fstype=btrfs --grow +btrfs none --label=fedora btrfs.007 +btrfs /home --subvol --name=home LABEL=fedora +btrfs / --subvol --name=root LABEL=fedora + +repo --name=fedora --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch +repo --name=updates --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=updates-released-f$releasever&arch=$basearch +url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch + +reboot + +##### begin package list ############################################# +%packages --instLangs=en + +# Include packages for the cloud-server-environment group +@^cloud-server-environment + +# Install the tracer dnf plugin to enable automatic reboots +# IF the user requests package updates +# AND requests a reboot +# AND the packages updated require a reboot. +# https://fedoraproject.org/wiki/Changes/Automatic_Cloud_Reboot_On_Updates +python3-dnf-plugin-tracer + +# Don't include the kernel toplevel package since it pulls in +# kernel-modules. We're happy for now with kernel-core. +-kernel +kernel-core + +# Don't include dracut-config-rescue. It will have dracut generate a +# "rescue" entry in the grub menu, but that also means there is a +# rescue kernel and initramfs that get created, which (currently) add +# about another 40MiB to the /boot/ partition. Also the "rescue" mode +# is generally not useful in the cloud. +-dracut-config-rescue + +# Plymouth provides a graphical boot animation. In the cloud we don't +# need a graphical boot animation. This also means anaconda won't put +# rhgb/quiet on kernel command line +-plymouth + +# Install qemu-guest-agent https://pagure.io/cloud-sig/issue/319 To +# improve the integration with OpenStack and other VM management +# systems (oVirt, KubeVirt). +qemu-guest-agent + + +# No need for firewalld for now. We don't have a firewall on by default. +-firewalld + +# Don't include the geolite2 databases, which end up with 66MiB +# in /usr/share/GeoIP +-geolite2-country +-geolite2-city + +# Specific to vagrant +dnf-yum +rsync +fuse-sshfs +%end +##### end package list ############################################### + + +##### begin kickstart post ########################################### +%post --erroronfail +#!/bin/bash +set -x +if [ -e /dev/vda ]; then + device="/dev/vda" +elif [ -e /dev/sda ]; then + device="/dev/sda" +else + echo "No device found" + exit 1 +fi + +virt="$(systemd-detect-virt)" +if [ "${virt}" == "microsoft" ]; then + dnf install -y hyperv-daemons + systemctl enable hpervfcopyd + systemctl enable hypervkvpd + systemctl enable hypervvssd +fi + +if [ "$(arch)" = "x86_64" ]; then +# Set up legacy BIOS boot if we booted from UEFI +grub2-install --target=i386-pc "${device}" +fi + +# Blivet sets pmbr_boot flag erroneously and we need to purge it +# otherwise it'll fail to boot +parted "${device}" disk_set pmbr_boot off + +# linux-firmware is installed by default and is quite large. As of mid 2020: +# Total download size: 97 M +# Installed size: 268 M +# So far we've been fine shipping without it so let's continue. +# More discussion about this in #1234504. +echo "Removing linux-firmware package." +rpm -e linux-firmware + +# See the systemd-random-seed.service man page that says: +# " It is recommended to remove the random seed from OS images intended +# for replication on multiple systems" +echo "Removing random-seed so it's not the same in every image." +rm -f /var/lib/systemd/random-seed + +echo "Import RPM GPG key" +releasever=$(rpm --eval '%{fedora}') +basearch=$(uname -i) +rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch + +echo "Zeroing out empty space." +# Create zeros file with nodatacow and no compression +touch /var/tmp/zeros +chattr +C /var/tmp/zeros +# This forces the filesystem to reclaim space from deleted files +dd bs=1M if=/dev/zero of=/var/tmp/zeros || : +echo "(Don't worry -- that out-of-space error was expected.)" +# Force sync to disk (Cf. https://pagure.io/cloud-sig/issue/340#comment-743430) +btrfs filesystem sync / +rm -f /var/tmp/zeros +btrfs filesystem sync / + +# When we build the image a networking config file gets left behind. +# Let's clean it up. +echo "Cleanup leftover networking configuration" +rm -f /etc/NetworkManager/system-connections/*.nmconnection +#*/ + +# Truncate the /etc/resolv.conf left over from NetworkManager during the +# kickstart. This causes delays in boot with cloud-init because the +# 192.168.122.1 DNS server cannot be reached. +truncate -s 0 /etc/resolv.conf + +# Clear machine-id on pre generated images +truncate -s 0 /etc/machine-id + +# Vagrant setup +sed -i 's,Defaults\\s*requiretty,Defaults !requiretty,' /etc/sudoers +echo 'vagrant ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant-nopasswd +sed -i 's/.*UseDNS.*/UseDNS no/' /etc/ssh/sshd_config + +cat > /etc/ssh/sshd_config.d/10-vagrant-insecure-rsa-key.conf <e ", + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0" +] +cd_files = [ + "distros/fedora/40/*.cfg" +] +vbox_os_type = "Fedora_64" diff --git a/distros/fedora/40/finalize.sh b/distros/fedora/40/finalize.sh new file mode 100644 index 0000000..c30e5a8 --- /dev/null +++ b/distros/fedora/40/finalize.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex + +dnf -y update + +dnf clean all diff --git a/distros/fedora/40/qemu.sh b/distros/fedora/40/qemu.sh new file mode 100644 index 0000000..569dc7c --- /dev/null +++ b/distros/fedora/40/qemu.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex + +dnf install -y qemu-guest-agent +systemctl start qemu-guest-agent +systemctl enable qemu-guest-agent diff --git a/distros/fedora/40/virtualbox.sh b/distros/fedora/40/virtualbox.sh new file mode 100644 index 0000000..209dab5 --- /dev/null +++ b/distros/fedora/40/virtualbox.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex + +dnf install -y virtualbox-guest-additions +systemctl start vboxservice +systemctl enable vboxservice diff --git a/distros/fedora/40/vmware.sh b/distros/fedora/40/vmware.sh new file mode 100644 index 0000000..058afc2 --- /dev/null +++ b/distros/fedora/40/vmware.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -ex + +dnf install -y open-vm-tools diff --git a/distros/fedora/supports.yml b/distros/fedora/supports.yml index 295db47..8155968 100644 --- a/distros/fedora/supports.yml +++ b/distros/fedora/supports.yml @@ -1,5 +1,6 @@ 37: # EOL -38: &supp +38: # EOL +39: &supp - arch: amd64 provider: qemu - arch: amd64 @@ -8,7 +9,7 @@ provider: vmware-iso - arch: amd64 provider: hyperv-iso -39: *supp +40: *supp rawhide: - arch: amd64 provider: qemu diff --git a/distros/rockylinux/9/amd64.pkrvars.hcl b/distros/rockylinux/9/amd64.pkrvars.hcl index fd49627..be8f547 100644 --- a/distros/rockylinux/9/amd64.pkrvars.hcl +++ b/distros/rockylinux/9/amd64.pkrvars.hcl @@ -1,8 +1,8 @@ distro = "rockylinux" version = "9" iso = { - url = "http://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9.3-x86_64-boot.iso" - checksum = "eb096f0518e310f722d5ebd4c69f0322df4fc152c6189f93c5c797dc25f3d2e1" + url = "https://download.rockylinux.org/pub/rocky/9.4/isos/x86_64/Rocky-9.4-x86_64-boot.iso" + checksum = "c7e95e3dba88a1f68fff8b7d4e66adf6f76ac4fba2e246a83c46ab79574c78a8" } boot_command = [ "e", diff --git a/distros/rockylinux/supports.yml b/distros/rockylinux/supports.yml index 533b6df..175fab4 100644 --- a/distros/rockylinux/supports.yml +++ b/distros/rockylinux/supports.yml @@ -1,4 +1,4 @@ -8: &supp +9: &supp - arch: amd64 provider: qemu - arch: amd64 @@ -7,4 +7,3 @@ provider: vmware-iso - arch: amd64 provider: hyperv-iso -9: *supp From 13f2dc79432c38898c9be1e450778be0b5e17d65 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 10:48:58 -0500 Subject: [PATCH 19/53] Try manually running uploads --- .gitlab-ci-build.yml.in | 14 ++------------ .gitlab-ci.yml | 9 +++++---- cache.sh | 2 +- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 13bef34..770b879 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -11,10 +11,6 @@ stages: - sources/build.pkr.hcl paths: - ./packer_config/ - - key: isofiles - paths: - - ./cache/*.iso - policy: pull rules: - if: $CI_COMMIT_TAG variables: @@ -46,6 +42,8 @@ stages: before_script: - packer init sources - echo $COMMAND + - mc alias add storage "localhost:9000" root "$MINIO_SECRET" + - mc get storage/isos script: - ls -l cache/ - echo $COMMAND > build.sh @@ -68,10 +66,6 @@ virtualbox-iso.amd64: tag: vbox parallel: matrix: [] - after_script: |- - if [ "$CI_JOB_STATUS" == "success" ]; then - xonsh test.xsh - fi hyperv-iso.amd64: extends: .build @@ -85,10 +79,6 @@ hyperv-iso.amd64: - echo $env:COMMAND > build.ps1 - echo 'if (-not $?) { throw "Error in build"}' >> build.ps1 - pwsh .\build.ps1 - after_script: |- - if ( "$CI_JOB_STATUS" -eq "success" ) { - xonsh test.xsh - } "Create release": stage: release diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 930f921..f57e488 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,13 +69,14 @@ downloads: - sources/build.pkr.hcl paths: - ./packer_config/ - - key: isofiles - paths: - - ./cache/*.iso - policy: pull-push script: - packer init sources - bash cache.sh + - mc alias add storage "localhost:9000" root "$MINIO_SECRET" + - | + for f in cache/*; do + mc put "$f" "storage/isos/$(basename "$f")" + end trigger: stage: trigger diff --git a/cache.sh b/cache.sh index f7fad98..dc42722 100755 --- a/cache.sh +++ b/cache.sh @@ -23,4 +23,4 @@ echo "Downloading ${files[*]}" curl --parallel -L "${files[@]}" popd -ls -l cache/ +ls -lh cache/ From c1156a93d3f759036a867950e19fbcc373339185 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 11:01:09 -0500 Subject: [PATCH 20/53] Fix invocations to minio-client --- .gitlab-ci-build.yml.in | 5 ++++- .gitlab-ci.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 770b879..247bf42 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -43,7 +43,10 @@ stages: - packer init sources - echo $COMMAND - mc alias add storage "localhost:9000" root "$MINIO_SECRET" - - mc get storage/isos + - mc ls storage/isos/cache + - file=$(echo var.iso.url | packer console -var-file="${distro}" -config-type=hcl2 sources/) + - mkdir cache + - mc get "storage/isos/$file" "$file" script: - ls -l cache/ - echo $COMMAND > build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f57e488..6deddea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,7 +75,7 @@ downloads: - mc alias add storage "localhost:9000" root "$MINIO_SECRET" - | for f in cache/*; do - mc put "$f" "storage/isos/$(basename "$f")" + mc put "$f" "storage/isos/$f" end trigger: From 9cf56f635fd9303d8affc797c8ad7a37a3c1935b Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 11:27:59 -0500 Subject: [PATCH 21/53] Correct the writing of files --- .gitlab-ci-build.yml.in | 2 +- .gitlab-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 247bf42..c8c1910 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -42,7 +42,7 @@ stages: before_script: - packer init sources - echo $COMMAND - - mc alias add storage "localhost:9000" root "$MINIO_SECRET" + - mc alias set storage "localhost:9000" root "$MINIO_SECRET" - mc ls storage/isos/cache - file=$(echo var.iso.url | packer console -var-file="${distro}" -config-type=hcl2 sources/) - mkdir cache diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6deddea..3fcb28d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,7 @@ downloads: script: - packer init sources - bash cache.sh - - mc alias add storage "localhost:9000" root "$MINIO_SECRET" + - mc alias set storage "localhost:9000" root "$MINIO_SECRET" - | for f in cache/*; do mc put "$f" "storage/isos/$f" From 91c56299ec946c5bf4ac8dec805bb29f2e570558 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 12:33:38 -0500 Subject: [PATCH 22/53] Use full URL --- .envrc | 1 + .gitlab-ci-build.yml.in | 2 +- .gitlab-ci.yml | 2 +- flake.lock | 194 ++++++++++++++++++++++++++++++++++++++++ flake.nix | 79 ++++++++++++++++ 5 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 .envrc create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index c8c1910..60ffece 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -42,7 +42,7 @@ stages: before_script: - packer init sources - echo $COMMAND - - mc alias set storage "localhost:9000" root "$MINIO_SECRET" + - mc alias set storage "http://localhost:9000" root "$MINIO_SECRET" - mc ls storage/isos/cache - file=$(echo var.iso.url | packer console -var-file="${distro}" -config-type=hcl2 sources/) - mkdir cache diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fcb28d..7255dd5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,7 @@ downloads: script: - packer init sources - bash cache.sh - - mc alias set storage "localhost:9000" root "$MINIO_SECRET" + - mc alias set storage "http://localhost:9000" root "$MINIO_SECRET" - | for f in cache/*; do mc put "$f" "storage/isos/$f" diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..45463b6 --- /dev/null +++ b/flake.lock @@ -0,0 +1,194 @@ +{ + "nodes": { + "devshell": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1717408969, + "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=", + "owner": "numtide", + "repo": "devshell", + "rev": "1ebbe68d57457c8cae98145410b164b5477761f4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1717285511, + "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-root": { + "locked": { + "lastModified": 1713493429, + "narHash": "sha256-ztz8JQkI08tjKnsTpfLqzWoKFQF4JGu2LRz8bkdnYUk=", + "owner": "srid", + "repo": "flake-root", + "rev": "bc748b93b86ee76e2032eecda33440ceb2532fcd", + "type": "github" + }, + "original": { + "owner": "srid", + "repo": "flake-root", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1704161960, + "narHash": "sha256-QGua89Pmq+FBAro8NriTuoO/wNaUtugt29/qqA8zeeM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63143ac2c9186be6d9da6035fa22620018c85932", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1717284937, + "narHash": "sha256-lIbdfCsf8LMFloheeE6N31+BMIeixqyQWbSr2vk79EQ=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1718870667, + "narHash": "sha256-jab3Kpc8O1z3qxwVsCMHL4+18n5Wy/HHKyu1fcsF7gs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9b10b8f00cb5494795e5f51b39210fed4d2b0748", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1708475490, + "narHash": "sha256-g1v0TsWBQPX97ziznfJdWhgMyMGtoBFs102xSYO4syU=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "0e74ca98a74bc7270d28838369593635a5db3260", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "process-compose": { + "locked": { + "lastModified": 1718031437, + "narHash": "sha256-+RrlkAVZx0QhyeHAGFJnjST+/7Dc3zsDU3zAKXoDXaI=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "9344fac44edced4c686721686a6ad904d067c546", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, + "root": { + "inputs": { + "devshell": "devshell", + "flake-parts": "flake-parts", + "flake-root": "flake-root", + "nixpkgs": "nixpkgs_2", + "process-compose": "process-compose", + "treefmt-nix": "treefmt-nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1718522839, + "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..6a17f39 --- /dev/null +++ b/flake.nix @@ -0,0 +1,79 @@ +# github.com/brianmcgee has lots of great ideas for going above and beyond for this +{ + description = "Greg's homepage"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + devshell.url = "github:numtide/devshell"; + flake-parts.url = "github:hercules-ci/flake-parts"; + flake-root.url = "github:srid/flake-root"; + process-compose.url = "github:Platonic-Systems/process-compose-flake"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + }; + + outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, ... }@inputs: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + inputs.devshell.flakeModule + inputs.flake-root.flakeModule + inputs.treefmt-nix.flakeModule + ]; + flake = { + # Universal things + }; + systems = [ + "x86_64-linux" + "aarch64-linux" + ]; + perSystem = { config, pkgs, self', system, ... }: let + lib = pkgs.lib; + in { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + config.allowUnfree = true; + }; + # https://numtide.github.io/devshell/intro.html + devshells.default = { + # These commands will be shown in the motd that users read + # upon entering the shell + commands = [ + ]; + }; + + # This is not special, this is just flakes + packages = let + src = ./.; + in { + cache = pkgs.writeShellApplication { + name = "cache-isos"; + runtimeInputs = with pkgs; [ + minio-client + packer + ]; + text = let + #getIso = t: pkgs.runCommand "t.txt" { + #nativeInputs = [ pkgs.packer ]; + #} "echo var.iso.url | ${pkgs.packer}/bin/packer console -var-file=\"${t}\" -config-type=hcl2 ${src}/sources/"; + #isos = lib.attrsets.mapAttrsToList (t: builtins.readFile (getIso t)) templates; + #processIsos = builtins.concatStringsSep "\n" (builtins.map (i: "echo ${i}") isos); + in '' + packer init sources/ + rm -f cache/*latest* cache/*Rawhide* + ''; + }; + }; + + # https://numtide.github.io/treefmt/ + treefmt.config = { + inherit (config.flake-root) projectRootFile; + package = pkgs.treefmt; + + programs = { + deadnix.enable = true; + }; + + settings.formatter = { + }; + }; + }; + }; +} From fc1d419506000b7b6273d9aeac6b429b8f620009 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 13:00:22 -0500 Subject: [PATCH 23/53] Tweak download script --- .gitignore | 1 + .gitlab-ci.yml | 8 +------- cache.sh | 6 ++++++ flake.nix | 23 ++++++++--------------- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 5012c47..db1ffb1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ serial-output* grub.cfg isolinux.cfg Vagrantfile +.direnv # Generated distros distros/fedora/rawhide/*.hcl diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7255dd5..3356321 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,13 +70,7 @@ downloads: paths: - ./packer_config/ script: - - packer init sources - - bash cache.sh - - mc alias set storage "http://localhost:9000" root "$MINIO_SECRET" - - | - for f in cache/*; do - mc put "$f" "storage/isos/$f" - end + - nix run ".#cache" trigger: stage: trigger diff --git a/cache.sh b/cache.sh index dc42722..e389341 100755 --- a/cache.sh +++ b/cache.sh @@ -24,3 +24,9 @@ curl --parallel -L "${files[@]}" popd ls -lh cache/ + +mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + +for f in cache/*; do + mc put "$f" "storage/isos/$f" +done diff --git a/flake.nix b/flake.nix index 6a17f39..463cf16 100644 --- a/flake.nix +++ b/flake.nix @@ -35,30 +35,23 @@ devshells.default = { # These commands will be shown in the motd that users read # upon entering the shell - commands = [ - ]; + commands = [ { + package = self'.packages.cache; + category = "utils"; + help = "Downloads ISO files and adds them to the S3 bucket"; + } ]; }; # This is not special, this is just flakes - packages = let - src = ./.; - in { + packages = { cache = pkgs.writeShellApplication { name = "cache-isos"; runtimeInputs = with pkgs; [ + curl minio-client packer ]; - text = let - #getIso = t: pkgs.runCommand "t.txt" { - #nativeInputs = [ pkgs.packer ]; - #} "echo var.iso.url | ${pkgs.packer}/bin/packer console -var-file=\"${t}\" -config-type=hcl2 ${src}/sources/"; - #isos = lib.attrsets.mapAttrsToList (t: builtins.readFile (getIso t)) templates; - #processIsos = builtins.concatStringsSep "\n" (builtins.map (i: "echo ${i}") isos); - in '' - packer init sources/ - rm -f cache/*latest* cache/*Rawhide* - ''; + text = builtins.readFile ./cache.sh; }; }; From 9bb997c9e7699ea08dcd67d6d41445b7b69213ea Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 13:09:42 -0500 Subject: [PATCH 24/53] Only run nix with Nix command --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3356321..3ac06e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,6 +63,8 @@ shellcheck: downloads: stage: cache retry: 2 + tags: + - nix cache: - key: files: From 04b625f9e49fd3a5e8f429037e3964e59bd6f166 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 16:23:24 -0500 Subject: [PATCH 25/53] Add an explicit cache restore --- .gitlab-ci-build.yml.in | 8 +------- .gitlab-ci.yml | 1 + cache-restore.sh | 5 +++++ flake.nix | 9 +++++++++ 4 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 cache-restore.sh diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 60ffece..aa785a8 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -40,13 +40,7 @@ stages: ${EXCEPT} sources before_script: - - packer init sources - - echo $COMMAND - - mc alias set storage "http://localhost:9000" root "$MINIO_SECRET" - - mc ls storage/isos/cache - - file=$(echo var.iso.url | packer console -var-file="${distro}" -config-type=hcl2 sources/) - - mkdir cache - - mc get "storage/isos/$file" "$file" + - nix run ".#cache-restore" script: - ls -l cache/ - echo $COMMAND > build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ac06e3..5b989d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,6 +72,7 @@ downloads: paths: - ./packer_config/ script: + - nix run ".#cache-restore" - nix run ".#cache" trigger: diff --git a/cache-restore.sh b/cache-restore.sh new file mode 100644 index 0000000..0d25425 --- /dev/null +++ b/cache-restore.sh @@ -0,0 +1,5 @@ +mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + +for f in $(mc ls --json storage/isos/cache | jq '.key' | tr -d '"'); do + mc get -q "storage/isos/cache/$f" "./cache/$f" +done diff --git a/flake.nix b/flake.nix index 463cf16..e31c605 100644 --- a/flake.nix +++ b/flake.nix @@ -53,6 +53,15 @@ ]; text = builtins.readFile ./cache.sh; }; + cache-restore = pkgs.writeShellApplication { + name = "cache-restore"; + runtimeInputs = with pkgs; [ + jq + minio-client + packer + ]; + text = builtins.readFile ./cache-restore.sh; + }; }; # https://numtide.github.io/treefmt/ From 3264070b84889ba1ed731c5f258e71cebdf5fdf7 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 21 Jun 2024 16:24:37 -0500 Subject: [PATCH 26/53] Explicitly call shell shebang --- cache-restore.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cache-restore.sh b/cache-restore.sh index 0d25425..8e1a5ff 100644 --- a/cache-restore.sh +++ b/cache-restore.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" for f in $(mc ls --json storage/isos/cache | jq '.key' | tr -d '"'); do From e25f08ee63b3eb8e135c7886f6a7791d4398d968 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Mon, 24 Jun 2024 22:40:48 -0500 Subject: [PATCH 27/53] Cat out the file first --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b989d1..048fad5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,7 @@ distros: - .gitlab-ci-build.yml script: |- xonsh ./build.xsh -a --list + cat .gitalb-ci-build.yml echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env validate: From 8f25c08be776bbd43dbc05d64c25364802b65c5c Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Mon, 24 Jun 2024 22:43:44 -0500 Subject: [PATCH 28/53] Cat out file somewhere else --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 048fad5..49ce53c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,6 @@ distros: - .gitlab-ci-build.yml script: |- xonsh ./build.xsh -a --list - cat .gitalb-ci-build.yml echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env validate: @@ -84,3 +83,5 @@ trigger: job: distros variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID + script: + - cat .gitalb-ci-build.yml From c5875f9943091a08b1baccb4674c1a8f2bb38691 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Mon, 24 Jun 2024 22:45:42 -0500 Subject: [PATCH 29/53] Remove cat command --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49ce53c..5b989d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -83,5 +83,3 @@ trigger: job: distros variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID - script: - - cat .gitalb-ci-build.yml From f138e5af3ef2b32a126145380076d8f8c7286249 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Mon, 24 Jun 2024 23:04:16 -0500 Subject: [PATCH 30/53] Nixify shellcheck Add shellcheck command to flake.nix and use it in the Gitlab --- .gitlab-ci.yml | 6 +++--- flake.nix | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b989d1..d125939 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,9 +51,9 @@ validate: shellcheck: stage: lint - script: |- - shellcheck --version - shellcheck -e SC2239 $(find . -name '*.sh') + tags: + - nix + script: nix run ".#shellcheck" # Downloads all the ISO files so that we have a single cache with all # of them present. This prevents a race condition where some of the diff --git a/flake.nix b/flake.nix index e31c605..a04ec48 100644 --- a/flake.nix +++ b/flake.nix @@ -39,6 +39,10 @@ package = self'.packages.cache; category = "utils"; help = "Downloads ISO files and adds them to the S3 bucket"; + } { + package = self'.packages.shellcheck; + category = "utils"; + help = "Run shellchecks"; } ]; }; @@ -62,6 +66,20 @@ ]; text = builtins.readFile ./cache-restore.sh; }; + shellcheck = pkgs.writeShellApplication { + name = "shellcheck"; + runtimeInputs = with pkgs; [ + shellcheck + ]; + text = '' + scripts=() + shopt -s globstar + for f in **/*.sh; do + scripts+=("$f") + done + shellcheck -e SC2239 "''${scripts[@]}" + ''; + }; }; # https://numtide.github.io/treefmt/ From 2d124c652ae4da7ff35b5ea9b72bd56b9a8062af Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Mon, 24 Jun 2024 23:13:53 -0500 Subject: [PATCH 31/53] Move validate step into Nix script --- .gitlab-ci.yml | 9 +++------ flake.nix | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d125939..d79abb2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,8 @@ distros: validate: stage: lint + tags: + - nix needs: - generate cache: @@ -42,12 +44,7 @@ validate: - sources/build.pkr.hcl paths: - ./packer_config/ - script: |- - set -exo pipefail - packer init sources - for vars in $(find distros/ -name '*.pkrvars.hcl'); do - packer validate -except=upload -var-file=${vars} sources/ - done + script: nix run ".#validate" shellcheck: stage: lint diff --git a/flake.nix b/flake.nix index a04ec48..897a912 100644 --- a/flake.nix +++ b/flake.nix @@ -43,6 +43,10 @@ package = self'.packages.shellcheck; category = "utils"; help = "Run shellchecks"; + } { + package = self'.packages.validate; + category = "utils"; + help = "Validate all the files"; } ]; }; @@ -80,6 +84,19 @@ shellcheck -e SC2239 "''${scripts[@]}" ''; }; + validate = pkgs.writeShellApplication { + name = "validate"; + runtimeInputs = with pkgs; [ + packer + ]; + text = '' + shopt -s globstar + packer init sources + for vars in distros/**/*.pkrvars.hcl; do + packer validate -except=upload "-var-file=''${vars}" sources/ + done + ''; + }; }; # https://numtide.github.io/treefmt/ From 832c901b9b21d4e8826c5dcc0db591d510e59428 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 16:00:57 -0500 Subject: [PATCH 32/53] Move almost everything over to Nix Eliminate Xonsh usage Eliminate usage of the supports.yml files for default.nix Point ISOs directly to MinIO URLs instead --- .gitignore | 1 + .gitlab-ci-build.yml.in | 9 +-- build.xsh | 93 ------------------------------ cache.sh | 2 +- del_boxes.xsh | 6 -- distros/almalinux/default.nix | 9 +++ distros/almalinux/supports.yml | 9 --- distros/alpine/default.nix | 5 ++ distros/alpine/supports.yml | 7 --- distros/archlinux/default.nix | 5 ++ distros/archlinux/supports.yml | 9 --- distros/centos-stream/default.nix | 9 +++ distros/centos-stream/supports.yml | 9 --- distros/centos/default.nix | 5 ++ distros/centos/supports.yml | 9 --- distros/debian/default.nix | 20 +++++++ distros/debian/supports.yml | 17 ------ distros/default.nix | 13 +++++ distros/fedora/default.nix | 21 +++++++ distros/fedora/supports.yml | 21 ------- distros/nixos/default.nix | 9 +++ distros/nixos/supports.yml | 10 ---- distros/opensuse/default.nix | 9 +++ distros/opensuse/supports.yml | 11 ---- distros/rockylinux/default.nix | 9 +++ distros/rockylinux/supports.yml | 9 --- distros/ubuntu/default.nix | 17 ++++++ distros/ubuntu/supports.yml | 13 ----- flake.nix | 70 +++++++++++++++++++--- lib/__init__.py | 0 lib/build.py | 40 ------------- lib/distro.py | 28 --------- lib/provider.py | 17 ------ lib/support.py | 23 -------- sources/inputs.pkr.hcl | 10 ++++ sources/qemu.pkr.hcl | 2 +- test.xsh | 88 ---------------------------- 37 files changed, 210 insertions(+), 434 deletions(-) delete mode 100755 build.xsh delete mode 100755 del_boxes.xsh create mode 100644 distros/almalinux/default.nix delete mode 100644 distros/almalinux/supports.yml create mode 100644 distros/alpine/default.nix delete mode 100644 distros/alpine/supports.yml create mode 100644 distros/archlinux/default.nix delete mode 100644 distros/archlinux/supports.yml create mode 100644 distros/centos-stream/default.nix delete mode 100644 distros/centos-stream/supports.yml create mode 100644 distros/centos/default.nix delete mode 100644 distros/centos/supports.yml create mode 100644 distros/debian/default.nix delete mode 100644 distros/debian/supports.yml create mode 100644 distros/default.nix create mode 100644 distros/fedora/default.nix delete mode 100644 distros/fedora/supports.yml create mode 100644 distros/nixos/default.nix delete mode 100644 distros/nixos/supports.yml create mode 100644 distros/opensuse/default.nix delete mode 100644 distros/opensuse/supports.yml create mode 100644 distros/rockylinux/default.nix delete mode 100644 distros/rockylinux/supports.yml create mode 100644 distros/ubuntu/default.nix delete mode 100644 distros/ubuntu/supports.yml delete mode 100644 lib/__init__.py delete mode 100644 lib/build.py delete mode 100644 lib/distro.py delete mode 100644 lib/provider.py delete mode 100644 lib/support.py delete mode 100755 test.xsh diff --git a/.gitignore b/.gitignore index db1ffb1..bbe3246 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ grub.cfg isolinux.cfg Vagrantfile .direnv +result # Generated distros distros/fedora/rawhide/*.hcl diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index aa785a8..3b07a05 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -39,10 +39,7 @@ stages: -var memory=4096 ${EXCEPT} sources - before_script: - - nix run ".#cache-restore" script: - - ls -l cache/ - echo $COMMAND > build.sh - bash build.sh || exit 1 @@ -53,7 +50,7 @@ qemu.amd64: only: "-only=qemu.amd64" tag: qemu parallel: - matrix: [] + matrix: QEMU_MATRIX virtualbox-iso.amd64: extends: .build @@ -62,7 +59,7 @@ virtualbox-iso.amd64: only: '-only="virtualbox-iso.amd64"' tag: vbox parallel: - matrix: [] + matrix: VBOX_MATRIX hyperv-iso.amd64: extends: .build @@ -71,7 +68,7 @@ hyperv-iso.amd64: only: '-only="hyperv-iso.amd64"' tag: hyperv parallel: - matrix: [] + matrix: HYPERV_MATRIX script: - echo $env:COMMAND > build.ps1 - echo 'if (-not $?) { throw "Error in build"}' >> build.ps1 diff --git a/build.xsh b/build.xsh deleted file mode 100755 index d129fe7..0000000 --- a/build.xsh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env xonsh -from pathlib import Path -import sys -sys.path.insert(0, str(Path(__file__).parent / "lib")) -from support import Support, BASE -import time -import json - -from argparse import ArgumentParser -from functools import reduce -from yaml import dump, Dumper, load, Loader - -source test.xsh - - -year, month, day, hour, minute, second, wday, jday, dst = time.localtime() -STAMP = f"build={year}.{month}.{day}.{hour}" - -support = Support() -parser = ArgumentParser("Build a box") -parser.add_argument("--distro", "-d", help="Name of the distro/ folder") -parser.add_argument("--version", "-v", help="Version of the distro to build") -parser.add_argument("--build", "-b", help="Name of the target build, e.g. qemu.amd64, virtualbox-iso.amd64") -parser.add_argument("--list", "-l", help="List all build targets", action="store_true") -parser.add_argument("--all", "-a", help="Build all applicable targets", action="store_true") -parser.add_argument("--upload", "-u", help="Force doing uploads", action="store_true") -parser.add_argument("--headless", help="Run builds headless", action="store_true") -parser.add_argument("--stamp", "-s", help="Time stamp for builds", default=STAMP) -args = parser.parse_args() - - -my_list = [] - - -# Now provide menus if we don't have defaults -def process_all(): - if args.distro: - process_distro(support.distros[args.distro]) - else: - for name, distro in support.distros.items(): - process_distro(distro) - -def process_distro(distro): - if args.version: - process_builds(distro.builds_for_version(args.version)) - else: - process_builds(distro.builds) - -def process_builds(builds): - for build in builds: - if args.build: - name, arch = args.build.split(".") - if build.provider.name == name and build.provider.arch == arch: - do_build(build) - else: - do_build(build) - -def do_build(build): - if args.headless: - headless = "headless=true" - else: - headless = "headless=false" - - if args.list: - my_list.append(build) - else: - if args.upload: - if "VAGRANT_CLOUD_TOKEN" not in ${...}: - print("You need to set your cloud token") - sys.exit(1) - packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) @(BASE / "sources") - else: - upload = "-except=upload" - # Skip building file if we already have built it - output = Path(__file__).parent / "output" / get_vagrant_provider(build.provider.name) / f"{build.distro}-{build.version}-{build.provider.arch}.box" - if output.exists(): - print(f"Skipping {build} as box already exists") - else: - packer build -var @(headless) -var @(args.stamp) @(build.var_file) @(build.only) -except=upload @(BASE / "sources") - -if args.all or args.list or (args.distro and args.version and args.build): - process_all() - if args.list: - with open(".gitlab-ci-build.yml.in", "r") as fp: - y = load(fp, Loader=Loader) - for item in my_list: - print(item) - if item.is_github: - y[ item.only[6:] ]["parallel"]["matrix"].append({"distro": item.var_path}) - with open(".gitlab-ci-build.yml", "w") as fp: - dump(y, fp, Dumper=Dumper) -else: - print("Doing menu stuff") diff --git a/cache.sh b/cache.sh index e389341..ae64694 100755 --- a/cache.sh +++ b/cache.sh @@ -4,7 +4,7 @@ set -eo pipefail files=() # These files are always out of date -rm -f cache/*latest* cache/*Rawhide* +rm -f cache/*latest* cache/*Rawhide* cache/openSUSE*Current* # shellcheck disable=SC2044 for vars in $(find distros/ -name '*.pkrvars.hcl'); do diff --git a/del_boxes.xsh b/del_boxes.xsh deleted file mode 100755 index 79cb6ed..0000000 --- a/del_boxes.xsh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -virsh -q vol-list default | awk '{print $1}' | xargs -r -L 1 virsh vol-delete --pool default -sudo virsh -q vol-list default | awk '{print $1}' | sudo xargs -r -L 1 virsh vol-delete --pool default -vagrant box list | grep virtualbox | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider virtualbox --force -vagrant box list | grep libvirt | awk '{print $1}' | sort -u | xargs -r -l vagrant box remove --provider libvirt --force diff --git a/distros/almalinux/default.nix b/distros/almalinux/default.nix new file mode 100644 index 0000000..a0837c6 --- /dev/null +++ b/distros/almalinux/default.nix @@ -0,0 +1,9 @@ +[ { + version = "8"; + arch = "amd64"; + eol = true; +} { + version = "9"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/almalinux/supports.yml b/distros/almalinux/supports.yml deleted file mode 100644 index 175fab4..0000000 --- a/distros/almalinux/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -9: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/alpine/default.nix b/distros/alpine/default.nix new file mode 100644 index 0000000..15cdeb4 --- /dev/null +++ b/distros/alpine/default.nix @@ -0,0 +1,5 @@ +[ { + version = "3.19.0"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/alpine/supports.yml b/distros/alpine/supports.yml deleted file mode 100644 index 8c45e08..0000000 --- a/distros/alpine/supports.yml +++ /dev/null @@ -1,7 +0,0 @@ -"3.19.0": - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/archlinux/default.nix b/distros/archlinux/default.nix new file mode 100644 index 0000000..4fecf6e --- /dev/null +++ b/distros/archlinux/default.nix @@ -0,0 +1,5 @@ +[ { + version = "rolling"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/archlinux/supports.yml b/distros/archlinux/supports.yml deleted file mode 100644 index 68b73d5..0000000 --- a/distros/archlinux/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -rolling: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/centos-stream/default.nix b/distros/centos-stream/default.nix new file mode 100644 index 0000000..a0837c6 --- /dev/null +++ b/distros/centos-stream/default.nix @@ -0,0 +1,9 @@ +[ { + version = "8"; + arch = "amd64"; + eol = true; +} { + version = "9"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/centos-stream/supports.yml b/distros/centos-stream/supports.yml deleted file mode 100644 index 175fab4..0000000 --- a/distros/centos-stream/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -9: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/centos/default.nix b/distros/centos/default.nix new file mode 100644 index 0000000..061eca4 --- /dev/null +++ b/distros/centos/default.nix @@ -0,0 +1,5 @@ +[ { + version = "7"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/centos/supports.yml b/distros/centos/supports.yml deleted file mode 100644 index 6409bc4..0000000 --- a/distros/centos/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -7: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso \ No newline at end of file diff --git a/distros/debian/default.nix b/distros/debian/default.nix new file mode 100644 index 0000000..f8a6e30 --- /dev/null +++ b/distros/debian/default.nix @@ -0,0 +1,20 @@ +[ { + version = "9"; + arch = "amd64"; + eol = true; +} { + version = "10"; + arch = "amd64"; + eol = false; +} { + version = "11"; + arch = "amd64"; + eol = false; +} { + version = "12"; + arch = "amd64"; + eol = false; + exclude = [ + "virtualbox-iso" + ]; +} ] diff --git a/distros/debian/supports.yml b/distros/debian/supports.yml deleted file mode 100644 index 9865444..0000000 --- a/distros/debian/supports.yml +++ /dev/null @@ -1,17 +0,0 @@ -10: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -11: *supp -12: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso \ No newline at end of file diff --git a/distros/default.nix b/distros/default.nix new file mode 100644 index 0000000..0fd1bf5 --- /dev/null +++ b/distros/default.nix @@ -0,0 +1,13 @@ +{ + almalinux = import ./almalinux; + alpine = import ./alpine; + archlinux = import ./archlinux; + centos = import ./centos; + centos-stream = import ./centos-stream; + debian = import ./debian; + fedora = import ./fedora; + nixos = import ./nixos; + opensuse = import ./opensuse; + rockylinux = import ./rockylinux; + ubuntu = import ./ubuntu; +} diff --git a/distros/fedora/default.nix b/distros/fedora/default.nix new file mode 100644 index 0000000..17da572 --- /dev/null +++ b/distros/fedora/default.nix @@ -0,0 +1,21 @@ +[ { + version = "37"; + arch = "amd64"; + eol = true; +} { + version = "38"; + arch = "amd64"; + eol = true; +} { + version = "39"; + arch = "amd64"; + eol = false; +} { + version = "40"; + arch = "amd64"; + eol = false; +} { + version = "rawhide"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/fedora/supports.yml b/distros/fedora/supports.yml deleted file mode 100644 index 8155968..0000000 --- a/distros/fedora/supports.yml +++ /dev/null @@ -1,21 +0,0 @@ -37: # EOL -38: # EOL -39: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -40: *supp -rawhide: - - arch: amd64 - provider: qemu - - arch: amd64 - provider: hyperv-iso - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso diff --git a/distros/nixos/default.nix b/distros/nixos/default.nix new file mode 100644 index 0000000..f9e89b8 --- /dev/null +++ b/distros/nixos/default.nix @@ -0,0 +1,9 @@ +[ { + version = "23.05"; + arch = "amd64"; + eol = true; +} { + version = "23.11"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/nixos/supports.yml b/distros/nixos/supports.yml deleted file mode 100644 index d6edc34..0000000 --- a/distros/nixos/supports.yml +++ /dev/null @@ -1,10 +0,0 @@ -23.05: # EOL -23.11: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/opensuse/default.nix b/distros/opensuse/default.nix new file mode 100644 index 0000000..01de93d --- /dev/null +++ b/distros/opensuse/default.nix @@ -0,0 +1,9 @@ +[ { + version = "15.5"; + arch = "amd64"; + eol = false; +} { + version = "tumbleweed"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/opensuse/supports.yml b/distros/opensuse/supports.yml deleted file mode 100644 index 06d713d..0000000 --- a/distros/opensuse/supports.yml +++ /dev/null @@ -1,11 +0,0 @@ -15.5: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso -15.6: -tumbleweed: *supp diff --git a/distros/rockylinux/default.nix b/distros/rockylinux/default.nix new file mode 100644 index 0000000..a0837c6 --- /dev/null +++ b/distros/rockylinux/default.nix @@ -0,0 +1,9 @@ +[ { + version = "8"; + arch = "amd64"; + eol = true; +} { + version = "9"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/rockylinux/supports.yml b/distros/rockylinux/supports.yml deleted file mode 100644 index 175fab4..0000000 --- a/distros/rockylinux/supports.yml +++ /dev/null @@ -1,9 +0,0 @@ -9: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso diff --git a/distros/ubuntu/default.nix b/distros/ubuntu/default.nix new file mode 100644 index 0000000..c0643db --- /dev/null +++ b/distros/ubuntu/default.nix @@ -0,0 +1,17 @@ +[ { + version = "18.04"; + arch = "amd64"; + eol = true; +} { + version = "20.04"; + arch = "amd64"; + eol = false; +} { + version = "22.04"; + arch = "amd64"; + eol = false; +} { + version = "23.10"; + arch = "amd64"; + eol = false; +} ] diff --git a/distros/ubuntu/supports.yml b/distros/ubuntu/supports.yml deleted file mode 100644 index 80525fb..0000000 --- a/distros/ubuntu/supports.yml +++ /dev/null @@ -1,13 +0,0 @@ -20.04: &supp - - arch: amd64 - provider: qemu - - arch: amd64 - provider: virtualbox-iso - - arch: amd64 - provider: vmware-iso - - arch: amd64 - provider: hyperv-iso - # linux-azure package broken in 20.04 -22.04: *supp -"23.10": *supp -#18.04: *supp # EOL diff --git a/flake.nix b/flake.nix index 897a912..5cb00e5 100644 --- a/flake.nix +++ b/flake.nix @@ -10,8 +10,30 @@ treefmt-nix.url = "github:numtide/treefmt-nix"; }; - outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, ... }@inputs: - flake-parts.lib.mkFlake { inherit inputs; } { + outputs = { devshell, flake-parts, flake-root, process-compose, treefmt-nix, nixpkgs, ... }@inputs: let + lib = nixpkgs.lib; + providers = [ + "qemu" + "virtualbox-iso" + "hyperv-iso" + ]; + distros = import ./distros; + # Inserts the distro name into each element in the list + insertNames = lib.attrsets.mapAttrsToList (distro: val: (builtins.map (x: x // { inherit distro; }) val)) distros; + # Flattens all the distros into a single list + flat = lib.lists.flatten insertNames; + # Combines providers list with flattened list + withProviders = lib.attrsets.mapCartesianProduct + ({d, provider}: d // { inherit provider; }) + { d = flat; provider = providers; }; + active = builtins.filter (x: (x?eol) && (!x.eol)) withProviders; + name = d: "-${d.distro}-${d.arch}-${d.provider}-${d.version}"; + distroFile = d: "distros/${d.distro}/${d.version}/${d.arch}.pkrvars.hcl"; + cmd = d: '' + version="''${1:-1}" + packer build -var "build=$version" -only=${d.provider}.${d.arch} -var-file=${distroFile d}''; + in flake-parts.lib.mkFlake { inherit inputs; } + { imports = [ inputs.devshell.flakeModule inputs.flake-root.flakeModule @@ -24,9 +46,8 @@ "x86_64-linux" "aarch64-linux" ]; - perSystem = { config, pkgs, self', system, ... }: let - lib = pkgs.lib; - in { + perSystem = { config, pkgs, self', system, ... }: + { _module.args.pkgs = import inputs.nixpkgs { inherit system; config.allowUnfree = true; @@ -47,11 +68,46 @@ package = self'.packages.validate; category = "utils"; help = "Validate all the files"; - } ]; + } ] ++ (builtins.map (d: { package = self'.packages."build${name d}"; category = "builds"; }) withProviders); + env = [{ + name = "EFI_DIR"; + value = "${pkgs.OVMF.fd}/FV/"; + }]; }; # This is not special, this is just flakes - packages = { + packages = let + base = { + runtimeInputs = with pkgs; [ + packer + qemu_full + ]; + }; + build = d: pkgs.writeShellApplication (base // { + name = "build${name d}"; + text = '' + set -x + ${cmd d} -except=upload -var headless=false sources/ + ''; + }); + upload = d: pkgs.writeShellApplication (base // { + name = "upload${name d}"; + text = '' + set -x + ${cmd d} sources/ + ''; + }); + builds = lib.lists.foldr (d: acc: acc // { "build${name d}" = (build d); }) {} withProviders; + uploads = lib.lists.foldr (d: acc: acc // { "upload${name d}" = (upload d); }) {} withProviders; + toYAML = x: (pkgs.formats.yaml x).generate; + qemu_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "qemu") active)); + vbox_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "virtualbox-iso") active)); + hyperv_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "hyperv-iso") active)); + in builds // uploads // { + continue = pkgs.writeText "continue.yml" + (builtins.replaceStrings ["QEMU_MATRIX" "VBOX_MATRIX" "HYPERV_MATRIX"] [qemu_list vbox_list hyperv_list] + (builtins.readFile ./.gitlab-ci-build.yml.in) + ); cache = pkgs.writeShellApplication { name = "cache-isos"; runtimeInputs = with pkgs; [ diff --git a/lib/__init__.py b/lib/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/build.py b/lib/build.py deleted file mode 100644 index a0fa1c2..0000000 --- a/lib/build.py +++ /dev/null @@ -1,40 +0,0 @@ -from provider import Provider - - -class Build: - def __init__(self, distro, version, provider: Provider): - self.distro: str = distro - self.version: str = str(version) - self.provider: Provider = provider - - @property - def var_file(self): - return f"-var-file={self.var_path}" - - @property - def matrix(self): - return { - "builder": str(self.provider), - "distro": self.var_path, - "runner": self.provider.runner, - } - - @property - def is_github(self) -> bool: - if self.provider.runner is not None: - return True - return False - - @property - def var_path(self): - return f"distros/{self.distro}/{self.version}/{self.provider.arch}.pkrvars.hcl" - - @property - def only(self): - return f"-only={self.provider}" - - def __str__(self) -> str: - return f"{self.distro} {self.version}: {self.provider}" - - def __repr__(self) -> str: - return str(self) diff --git a/lib/distro.py b/lib/distro.py deleted file mode 100644 index bd3bdfa..0000000 --- a/lib/distro.py +++ /dev/null @@ -1,28 +0,0 @@ -from provider import Provider -from build import Build - - -class Distro: - def __init__(self, name: str, support: map): - self.name = name - self.builds = [] - for version, matrix in support.items(): - if not matrix: - continue - for support in matrix: - provider = Provider(support["provider"], support["arch"]) - build = Build(self.name, version, provider) - self.builds.append(build) - - def __str__(self) -> str: - return f"{self.name}" - - def __repr__(self) -> str: - return f"{self.name}: {' '.join(self.versions)}" - - @property - def versions(self) -> set[str]: - return set([b.version for b in self.builds]) - - def builds_for_version(self, version: str) -> set[Build]: - return set(filter(lambda x: x.version == version, self.builds)) diff --git a/lib/provider.py b/lib/provider.py deleted file mode 100644 index 60f2bd4..0000000 --- a/lib/provider.py +++ /dev/null @@ -1,17 +0,0 @@ -class Provider: - def __init__(self, name, arch): - self.name = name - self.arch = arch - - def __str__(self) -> str: - return f"{self.name}.{self.arch}" - - @property - def runner(self) -> str: - if self.name == "qemu": - return "qemu" - elif self.name.startswith("virtualbox"): - return "virtualbox" - elif self.name.startswith("hyperv"): - return "hyperv" - return None diff --git a/lib/support.py b/lib/support.py deleted file mode 100644 index 26571cb..0000000 --- a/lib/support.py +++ /dev/null @@ -1,23 +0,0 @@ -from distro import Distro -from provider import Provider -from pathlib import Path -from yaml import load, Loader - - -BASE = Path(__file__).parent.parent.resolve() - - -class Support: - def __init__(self): - self.base = BASE - self.distros = self.__get_distros() - - def __get_distros(self) -> map: - distros = {} - distros_dir = self.base / "distros" - for f in distros_dir.glob("**/supports.yml"): - name = f.parent.name - with open(f, "r") as fp: - support_yml = load(fp, Loader=Loader) - distros[name] = Distro(name, support_yml) - return distros diff --git a/sources/inputs.pkr.hcl b/sources/inputs.pkr.hcl index 711ff8e..7234a1d 100644 --- a/sources/inputs.pkr.hcl +++ b/sources/inputs.pkr.hcl @@ -109,6 +109,16 @@ variable "vbox_os_type" { default = "Linux_64" } +variable "storage_url" { + type = string + default = env("STORAGE_URL") +} + +variable "minio_secret" { + type = string + default = env("MINIO_SECRET") +} + locals { name = "${var.distro}-${var.version}" description = templatefile("../README.box.md", { diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index 0cc7e2c..9fe6366 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -4,7 +4,7 @@ variable "qemu_accelerator" { } source "qemu" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}" ] + iso_urls = [ "s3::${var.storage_url}isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/libvirt/${local.name}" diff --git a/test.xsh b/test.xsh deleted file mode 100755 index 7259672..0000000 --- a/test.xsh +++ /dev/null @@ -1,88 +0,0 @@ -#!env xonsh -# vim: set ft=xonsh: -import argparse -import pathlib -import re -import sys -import time -from shutil import rmtree - - -def get_vagrant_provider(provider): - "Vagrant provider is not always the same as Packer provider" - if provider.startswith("qemu"): - return "libvirt" - elif provider.startswith("virtualbox"): - return "virtualbox" - elif provider.startswith("vmware"): - return "vmware" - elif provider.startswith("hyperv"): - return "hyperv" - else: - return provider - - -def get_providers(): - "List of providers as defined in sources/" - providers: set[str] = { - "qemu.amd64" - "virtualbox-iso.amd64", - "vmware-iso.amd64" - } - return providers - - -def get_parser(args, unsupported=set()): - parser = argparse.ArgumentParser("Builder") - parser.add_argument("--headless", action="store_true") - parser.add_argument("--upload", action="store_true") - parser.add_argument("--provider", action="append") - parsed = parser.parse_args(args) - opts = {} - if parsed.headless: - opts["headless"] = "headless=true" - else: - opts["headless"] = "headless=false" - if parsed.upload: - opts["upload"] = "" - else: - opts["upload"] = "-except=upload" - if parsed.provider: - opts["providers"] = set(parsed.provider) - else: - opts["providers"] = get_providers() - # Filter providers - opts["providers"] -= unsupported - return opts - - -def main(): - # Equivalent of `set -e` in Bash - $RAISE_SUBPROC_ERROR = True - if len(sys.argv) == 2: - dry_run = True - else: - dry_run = False - for box in g`output/**/*.box`: - p = pathlib.Path(box) - with open("Vagrantfile.in", "r") as fr: - content = fr.read() - content = content.replace("@@BOX@@", p.resolve().as_uri()) - content = content.replace("@@BOX_NAME@@", p.name) - with open("Vagrantfile", "w") as fw: - fw.write(content) - if not dry_run: - if str(p.parent.name) in ["virtualbox", "libvirt", "vmware", "hyperv"]: - vagrant --provider @(p.parent.name) box add @(box) --name @(p.name) --force - vagrant up --provider @(p.parent.name) - else: - print(f"Now whatcha wanna go and do that for? {box}") - continue - vagrant destroy -f - vag_folder = pathlib.Path(".vagrant") - if vag_folder.exists(): - rmtree(vag_folder) - - -if __name__ == '__main__': - main() From 7f3b04da76578eb5141e63c3a975d8ebd329d209 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 16:03:13 -0500 Subject: [PATCH 33/53] Update distros to use nix --- .gitlab-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d79abb2..fda6189 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,9 +28,12 @@ distros: dotenv: build.env paths: - .gitlab-ci-build.yml - script: |- - xonsh ./build.xsh -a --list - echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env + tags: + - nix + script: + - nix build ".#continue" + - cp result .gitlab-ci-build.yml + - cat result validate: stage: lint From eebc19c81c150788609ac0aeae546a9511129247 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 16:17:33 -0500 Subject: [PATCH 34/53] Fix fetch URLs --- sources/hyperv-iso.pkr.hcl | 2 +- sources/qemu.pkr.hcl | 2 +- sources/virtualbox-iso.pkr.hcl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sources/hyperv-iso.pkr.hcl b/sources/hyperv-iso.pkr.hcl index f3f3adf..fd4ba7e 100644 --- a/sources/hyperv-iso.pkr.hcl +++ b/sources/hyperv-iso.pkr.hcl @@ -1,5 +1,5 @@ source "hyperv-iso" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}" ] + iso_urls = [ "s3::${var.storage_url}/isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/hyperv/${local.name}" diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index 9fe6366..c579021 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -4,7 +4,7 @@ variable "qemu_accelerator" { } source "qemu" "amd64" { - iso_urls = [ "s3::${var.storage_url}isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_urls = [ "s3::${var.storage_url}/isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/libvirt/${local.name}" diff --git a/sources/virtualbox-iso.pkr.hcl b/sources/virtualbox-iso.pkr.hcl index 37b2c74..541e207 100644 --- a/sources/virtualbox-iso.pkr.hcl +++ b/sources/virtualbox-iso.pkr.hcl @@ -1,5 +1,5 @@ source "virtualbox-iso" "amd64" { - iso_urls = [ "cache/${basename(var.iso.url)}" ] + iso_urls = [ "s3::${var.storage_url}/isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] iso_target_path = "cache/${basename(var.iso.url)}" iso_checksum = var.iso.checksum output_directory = "output/virtualbox/${local.name}" From 31f14a5e8fd89463acd6f565e85dbfd5b4f552c2 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 16:33:41 -0500 Subject: [PATCH 35/53] Start downloads right away There is no need to wait until after validation and linting is done before beginning the download caching, and it definitely does not need its own portion of the cycle. Move it to the beginning of the generate phase, so it gets started up right away. --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fda6189..58ad4c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,6 @@ stages: - generate - lint - - cache - trigger variables: @@ -61,7 +60,7 @@ shellcheck: # others were disappearing (e.g. daily builds for Rawhide) before we # could get to them downloads: - stage: cache + stage: generate retry: 2 tags: - nix From bc1f23a47b6086cc8b1ab2d83a682dad5d76293b Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 20:57:47 -0500 Subject: [PATCH 36/53] Better cache file naming --- cache.sh | 32 -------------------------- flake.nix | 10 +-------- nix/cache.nix | 41 ++++++++++++++++++++++++++++++++++ sources/hyperv-iso.pkr.hcl | 4 ++-- sources/qemu.pkr.hcl | 4 ++-- sources/virtualbox-iso.pkr.hcl | 4 ++-- 6 files changed, 48 insertions(+), 47 deletions(-) delete mode 100755 cache.sh create mode 100644 nix/cache.nix diff --git a/cache.sh b/cache.sh deleted file mode 100755 index ae64694..0000000 --- a/cache.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -files=() - -# These files are always out of date -rm -f cache/*latest* cache/*Rawhide* cache/openSUSE*Current* - -# shellcheck disable=SC2044 -for vars in $(find distros/ -name '*.pkrvars.hcl'); do - file=$(echo var.iso.url | packer console -var-file="${vars}" -config-type=hcl2 sources/) - base="$(basename "${file}")" - if [ -e "cache/${base}" ]; then - echo "Found cache/${base} already" - else - files+=("--retry" "3" "--retry-all-errors" "-O" "${file}") - fi -done - -mkdir -p cache -pushd cache -echo "Downloading ${files[*]}" -curl --parallel -L "${files[@]}" -popd - -ls -lh cache/ - -mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" - -for f in cache/*; do - mc put "$f" "storage/isos/$f" -done diff --git a/flake.nix b/flake.nix index 5cb00e5..e2ebdf8 100644 --- a/flake.nix +++ b/flake.nix @@ -108,15 +108,7 @@ (builtins.replaceStrings ["QEMU_MATRIX" "VBOX_MATRIX" "HYPERV_MATRIX"] [qemu_list vbox_list hyperv_list] (builtins.readFile ./.gitlab-ci-build.yml.in) ); - cache = pkgs.writeShellApplication { - name = "cache-isos"; - runtimeInputs = with pkgs; [ - curl - minio-client - packer - ]; - text = builtins.readFile ./cache.sh; - }; + cache = pkgs.callPackage (import ./nix/cache.nix) { inherit active distroFile; }; cache-restore = pkgs.writeShellApplication { name = "cache-restore"; runtimeInputs = with pkgs; [ diff --git a/nix/cache.nix b/nix/cache.nix new file mode 100644 index 0000000..33ea53f --- /dev/null +++ b/nix/cache.nix @@ -0,0 +1,41 @@ +{ + writeShellApplication, + active, + distroFile, + + curl, + minio-client, + packer, + + ... +}: + +let + inherit (builtins) concatStringsSep map; +in writeShellApplication { + name = "cache-isos"; + runtimeInputs = [ + curl + minio-client + packer + ]; + text = (concatStringsSep "\n" ( + map (d: '' + url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) + sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) + if [ ! -e "cache/$sha.iso" ]; then + curl --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" + else + echo "Skipping ${d.distro} - cache/$sha.iso" + fi + '') active + )) + '' + ls -lh cache/ + + mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + + for f in cache/*; do + mc put "$f" "storage/isos/$f" + done + ''; +} diff --git a/sources/hyperv-iso.pkr.hcl b/sources/hyperv-iso.pkr.hcl index fd4ba7e..0ce8ff9 100644 --- a/sources/hyperv-iso.pkr.hcl +++ b/sources/hyperv-iso.pkr.hcl @@ -1,6 +1,6 @@ source "hyperv-iso" "amd64" { - iso_urls = [ "s3::${var.storage_url}/isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] - iso_target_path = "cache/${basename(var.iso.url)}" + iso_urls = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_target_path = "cache/${var.iso.checksum}.iso" iso_checksum = var.iso.checksum output_directory = "output/hyperv/${local.name}" diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index c579021..dffafc8 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -4,8 +4,8 @@ variable "qemu_accelerator" { } source "qemu" "amd64" { - iso_urls = [ "s3::${var.storage_url}/isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] - iso_target_path = "cache/${basename(var.iso.url)}" + iso_urls = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_target_path = "cache/${var.iso.checksum}.iso" iso_checksum = var.iso.checksum output_directory = "output/libvirt/${local.name}" diff --git a/sources/virtualbox-iso.pkr.hcl b/sources/virtualbox-iso.pkr.hcl index 541e207..e2c9a56 100644 --- a/sources/virtualbox-iso.pkr.hcl +++ b/sources/virtualbox-iso.pkr.hcl @@ -1,6 +1,6 @@ source "virtualbox-iso" "amd64" { - iso_urls = [ "s3::${var.storage_url}/isos/cache/${basename(var.iso.url)}?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] - iso_target_path = "cache/${basename(var.iso.url)}" + iso_urls = [ "s3::${var.storage_url}/isos/cache/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" ] + iso_target_path = "cache/${var.iso.checksum}.iso" iso_checksum = var.iso.checksum output_directory = "output/virtualbox/${local.name}" From edffd23802620f12b7e20ba4e59d9a24e825255e Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 21:50:10 -0500 Subject: [PATCH 37/53] Have cache actually check cache Cache has not been hitting the actual cache bits, so now we will tell it to reach out to the cache server locally and check for the ISO to be present in the bucket before we run the curl at all --- nix/cache.nix | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/nix/cache.nix b/nix/cache.nix index 33ea53f..5db075d 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -19,23 +19,18 @@ in writeShellApplication { minio-client packer ]; - text = (concatStringsSep "\n" ( + text = '' + mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + '' + (concatStringsSep "\n" ( map (d: '' url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - if [ ! -e "cache/$sha.iso" ]; then + if [ -z "$(mc ls storage/isos/cache/ | grep $sha)" ]; then curl --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" + mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso" else echo "Skipping ${d.distro} - cache/$sha.iso" fi '') active - )) + '' - ls -lh cache/ - - mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" - - for f in cache/*; do - mc put "$f" "storage/isos/$f" - done - ''; + )); } From 810008b1ea820eb3053c76e8a591db16eccdc397 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 21:51:54 -0500 Subject: [PATCH 38/53] Skip restore step With the new cache setup, there is no need for me to run the cache restore before doing a download. However, it is now once again necessary that download begin AFTER the dynamic scripts have already executed their bits --- .gitlab-ci-build.yml.in | 4 ++-- .gitlab-ci.yml | 32 +++++++++++++++----------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in index 3b07a05..cf9f0cf 100644 --- a/.gitlab-ci-build.yml.in +++ b/.gitlab-ci-build.yml.in @@ -20,7 +20,7 @@ stages: EXCEPT: '-except=upload' needs: - pipeline: $PARENT_PIPELINE_ID - job: distros + job: continue artifacts: true - pipeline: $PARENT_PIPELINE_ID job: generate @@ -82,7 +82,7 @@ hyperv-iso.amd64: script: echo "Tagging as released" needs: - pipeline: $PARENT_PIPELINE_ID - job: distros + job: continue artifacts: true release: tag_name: '$CI_COMMIT_TAG' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58ad4c3..b246230 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ stages: - generate - - lint + - check - trigger variables: @@ -20,7 +20,7 @@ generate: bash "./${vars}" done -distros: +continue: stage: generate artifacts: reports: @@ -34,8 +34,14 @@ distros: - cp result .gitlab-ci-build.yml - cat result +shellcheck: + stage: generate + tags: + - nix + script: nix run ".#shellcheck" + validate: - stage: lint + stage: check tags: - nix needs: @@ -48,22 +54,15 @@ validate: - ./packer_config/ script: nix run ".#validate" -shellcheck: - stage: lint - tags: - - nix - script: nix run ".#shellcheck" - -# Downloads all the ISO files so that we have a single cache with all -# of them present. This prevents a race condition where some of the -# files were being downloaded multiple times due to parallel jobs, and -# others were disappearing (e.g. daily builds for Rawhide) before we -# could get to them +# Makes sure that all of the ISO files we are looking for are +# present on the S3 buckets that we use downloads: - stage: generate + stage: check retry: 2 tags: - nix + needs: + - generate cache: - key: files: @@ -71,7 +70,6 @@ downloads: paths: - ./packer_config/ script: - - nix run ".#cache-restore" - nix run ".#cache" trigger: @@ -79,6 +77,6 @@ trigger: trigger: include: - artifact: .gitlab-ci-build.yml - job: distros + job: continue variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID From 107922f3d0609d3f2184507c7d32c83eaf53bd95 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 22:04:16 -0500 Subject: [PATCH 39/53] Suppress Shellcheck errors --- nix/cache.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nix/cache.nix b/nix/cache.nix index 5db075d..2d57d57 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -25,7 +25,8 @@ in writeShellApplication { map (d: '' url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - if [ -z "$(mc ls storage/isos/cache/ | grep $sha)" ]; then + # shellcheck disable=SC2143 + if [ -z "$(mc ls storage/isos/cache/ | grep "$sha")" ]; then curl --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso" else From ff31288530e0629734ce2cc46e368027d08be042 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 22:09:54 -0500 Subject: [PATCH 40/53] Create director before using --- nix/cache.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/cache.nix b/nix/cache.nix index 2d57d57..a7f659a 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -21,6 +21,7 @@ in writeShellApplication { ]; text = '' mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + mkdir -p cache '' + (concatStringsSep "\n" ( map (d: '' url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) From 1629effd5179842545e81ffb962dea549cb821bd Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Wed, 26 Jun 2024 23:13:13 -0500 Subject: [PATCH 41/53] Tell cURL to follow redirects --- nix/cache.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/cache.nix b/nix/cache.nix index a7f659a..2305640 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -28,7 +28,7 @@ in writeShellApplication { sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) # shellcheck disable=SC2143 if [ -z "$(mc ls storage/isos/cache/ | grep "$sha")" ]; then - curl --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" + curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso" else echo "Skipping ${d.distro} - cache/$sha.iso" From e6804fefc36c48df6cacca66ea817aab5fd4046a Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 09:51:44 -0500 Subject: [PATCH 42/53] Only restore files that don't exist --- cache-restore.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cache-restore.sh b/cache-restore.sh index 8e1a5ff..b5d8208 100644 --- a/cache-restore.sh +++ b/cache-restore.sh @@ -2,5 +2,9 @@ mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" for f in $(mc ls --json storage/isos/cache | jq '.key' | tr -d '"'); do - mc get -q "storage/isos/cache/$f" "./cache/$f" + if [ ! -e "./cache/$f" ]; then + mc get -q "storage/isos/cache/$f" "./cache/$f" + else + echo "Skipping already present: $f" + fi done From c24e15d04e8f30c00e842d86799be1af1d7e6ade Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 09:52:25 -0500 Subject: [PATCH 43/53] Move more commands into nix --- .gitlab-ci.yml | 18 ++++-------------- flake.nix | 49 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b246230..640d1e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,16 +9,15 @@ variables: default: image: registry.thehellings.com/greg/ci-images/vm-test:latest + tags: + - nix generate: stage: generate artifacts: paths: - distros/**/*.pkrvars.hcl - script: |- - for vars in $(find distros/ -name '*.pkrvars.hcl.sh'); do - bash "./${vars}" - done + script: nix build ".#generate" continue: stage: generate @@ -27,8 +26,6 @@ continue: dotenv: build.env paths: - .gitlab-ci-build.yml - tags: - - nix script: - nix build ".#continue" - cp result .gitlab-ci-build.yml @@ -36,14 +33,10 @@ continue: shellcheck: stage: generate - tags: - - nix script: nix run ".#shellcheck" validate: stage: check - tags: - - nix needs: - generate cache: @@ -59,8 +52,6 @@ validate: downloads: stage: check retry: 2 - tags: - - nix needs: - generate cache: @@ -69,8 +60,7 @@ downloads: - sources/build.pkr.hcl paths: - ./packer_config/ - script: - - nix run ".#cache" + script: nix run ".#cache" trigger: stage: trigger diff --git a/flake.nix b/flake.nix index e2ebdf8..0febf82 100644 --- a/flake.nix +++ b/flake.nix @@ -60,6 +60,10 @@ package = self'.packages.cache; category = "utils"; help = "Downloads ISO files and adds them to the S3 bucket"; + } { + package = self'.packages.generate; + category = "utils"; + help = "Generate dynamic HCL files"; } { package = self'.packages.shellcheck; category = "utils"; @@ -68,6 +72,10 @@ package = self'.packages.validate; category = "utils"; help = "Validate all the files"; + } { + package = pkgs.vagrant; + category = "utils"; + help = "Vagrant to help with testing"; } ] ++ (builtins.map (d: { package = self'.packages."build${name d}"; category = "builds"; }) withProviders); env = [{ name = "EFI_DIR"; @@ -75,31 +83,37 @@ }]; }; - # This is not special, this is just flakes +# This is not special, this is just flakes packages = let base = { runtimeInputs = with pkgs; [ packer - qemu_full + qemu_full + vagrant ]; }; - build = d: pkgs.writeShellApplication (base // { + build = d: let + vp = x: if x.provider == "qemu" then "libvirt" + else if x.provider == "virtualbox-iso" then "virtualbox" + else if x.provider == "hyperv-iso" then "hyperv" + else "unknown"; + in pkgs.writeShellApplication (base // { name = "build${name d}"; text = '' - set -x - ${cmd d} -except=upload -var headless=false sources/ + set -x + ${cmd d} -except=upload -var headless=false sources/ + vagrant box add --provider ${vp d} ./output/${vp d}/${d.distro}-${d.version}.box --name test_box --force ''; - }); + }); upload = d: pkgs.writeShellApplication (base // { name = "upload${name d}"; text = '' - set -x - ${cmd d} sources/ + set -x + ${cmd d} sources/ ''; - }); + }); builds = lib.lists.foldr (d: acc: acc // { "build${name d}" = (build d); }) {} withProviders; uploads = lib.lists.foldr (d: acc: acc // { "upload${name d}" = (upload d); }) {} withProviders; - toYAML = x: (pkgs.formats.yaml x).generate; qemu_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "qemu") active)); vbox_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "virtualbox-iso") active)); hyperv_list = builtins.toJSON (builtins.map (d: { distro = distroFile d; }) (builtins.filter (x: x.provider == "hyperv-iso") active)); @@ -118,6 +132,21 @@ ]; text = builtins.readFile ./cache-restore.sh; }; + generate = pkgs.writeShellApplication { + name = "generate"; + runtimeInputs = with pkgs; [ + bash + curl + jq + pup + ]; + text = '' + shopt -s globstar + for vars in distros/**/*.pkrvars.hcl.sh; do + bash "./''${vars}" + done + ''; + }; shellcheck = pkgs.writeShellApplication { name = "shellcheck"; runtimeInputs = with pkgs; [ From c34dd1995e62ed1a4f00857548f44e698e1d697a Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 09:52:55 -0500 Subject: [PATCH 44/53] Fix Rawhide install --- distros/fedora/rawhide/amd64-ks.cfg | 4 +--- distros/fedora/rawhide/qemu.sh | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/distros/fedora/rawhide/amd64-ks.cfg b/distros/fedora/rawhide/amd64-ks.cfg index ccbb7d9..6d5fa77 100644 --- a/distros/fedora/rawhide/amd64-ks.cfg +++ b/distros/fedora/rawhide/amd64-ks.cfg @@ -1,5 +1,5 @@ user --name=vagrant --password=vagrant -rootpw vagrant +rootpw --plaintext vagrant text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 lang en_US.UTF-8 @@ -7,7 +7,6 @@ keyboard us timezone --utc Etc/UTC selinux --enforcing -rootpw --lock --iscrypted locked firewall --disabled @@ -77,7 +76,6 @@ qemu-guest-agent -geolite2-city # Specific to vagrant -dnf-yum rsync fuse-sshfs %end diff --git a/distros/fedora/rawhide/qemu.sh b/distros/fedora/rawhide/qemu.sh index 569dc7c..e44c536 100644 --- a/distros/fedora/rawhide/qemu.sh +++ b/distros/fedora/rawhide/qemu.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash set -ex -dnf install -y qemu-guest-agent systemctl start qemu-guest-agent systemctl enable qemu-guest-agent From 7b77d12a8bc7beb3c6dbf8da2fed20e1fd2da4b5 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 09:53:18 -0500 Subject: [PATCH 45/53] Make pushing optional --- nix/cache.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nix/cache.nix b/nix/cache.nix index 2305640..daadc80 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -20,7 +20,10 @@ in writeShellApplication { packer ]; text = '' - mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + push="''${1:-yes}" + if [ "$push" == "yes" ]; then + mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" + fi mkdir -p cache '' + (concatStringsSep "\n" ( map (d: '' @@ -29,7 +32,9 @@ in writeShellApplication { # shellcheck disable=SC2143 if [ -z "$(mc ls storage/isos/cache/ | grep "$sha")" ]; then curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" - mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso" + if [ "$push" == "yes" ]; then + mc put "cache/$sha.iso" "storage/isos/cache/$sha.iso" + fi else echo "Skipping ${d.distro} - cache/$sha.iso" fi From 9f2175d25fe717942ebdea23577cf00595268a4e Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 09:53:36 -0500 Subject: [PATCH 46/53] Go back to making serial file --- sources/qemu.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index dffafc8..ffcfaa9 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -31,7 +31,7 @@ source "qemu" "amd64" { efi_firmware_vars = "${var.efi_dir}OVMF_VARS.fd" qemuargs = [ - #["-chardev", "stdio,id=char0,logfile=serial-output-qemu-${var.arch}-${var.distro}-${var.version},signal=off"], + ["-chardev", "stdio,id=char0,logfile=serial-output-qemu-${var.arch}-${var.distro}-${var.version},signal=off"], ["-chardev", "socket,path=${local.name}.qga.sock,server=on,wait=off,id=qga0"], ["-device", "virtio-serial"], ["-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"], From 105240e1b314d2b162ae7e890755fd97cd7a57fd Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 09:59:05 -0500 Subject: [PATCH 47/53] Run, don't just build, the generate script --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 640d1e5..5ce6a85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ generate: artifacts: paths: - distros/**/*.pkrvars.hcl - script: nix build ".#generate" + script: nix run ".#generate" continue: stage: generate From 96da609d24ebe44edd23de29f6426fc6a8238991 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Fri, 28 Jun 2024 22:37:38 -0500 Subject: [PATCH 48/53] Try to push tumbleweed builds --- distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh | 2 +- distros/opensuse/tumbleweed/http/autoinst.xml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh index 7193d68..191c91b 100755 --- a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh +++ b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh @@ -2,7 +2,7 @@ name="${0%.sh}" -base="https://download.opensuse.org/tumbleweed/iso/openSUSE-Tumbleweed-NET-x86_64-Current.iso" +base="https://download.opensuse.org/tumbleweed/iso/openSUSE-Tumbleweed-DVD-x86_64-Current.iso" # Working directory tmp="$(mktemp -d)" diff --git a/distros/opensuse/tumbleweed/http/autoinst.xml b/distros/opensuse/tumbleweed/http/autoinst.xml index 68f4eec..774fa71 100644 --- a/distros/opensuse/tumbleweed/http/autoinst.xml +++ b/distros/opensuse/tumbleweed/http/autoinst.xml @@ -8,6 +8,7 @@ false true + true @@ -120,4 +121,4 @@ root ALL=(ALL) NOPASSWD: ALL https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant.pub - \ No newline at end of file + From 24e292ca6f5983eaa531dc4d5fa87cea4adaf7ff Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 29 Jun 2024 15:48:29 -0500 Subject: [PATCH 49/53] Simplify F39 kickstart --- distros/fedora/39/amd64-ks.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/distros/fedora/39/amd64-ks.cfg b/distros/fedora/39/amd64-ks.cfg index 90ed9f8..ba67610 100644 --- a/distros/fedora/39/amd64-ks.cfg +++ b/distros/fedora/39/amd64-ks.cfg @@ -1,5 +1,5 @@ user --name=vagrant --password=vagrant -rootpw vagrant +rootpw --plaintext vagrant text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 lang en_US.UTF-8 @@ -7,7 +7,6 @@ keyboard us timezone --utc Etc/UTC selinux --enforcing -rootpw --lock --iscrypted locked firewall --disabled From e91329f46f89483b9c599049b12584c513e60e7b Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 29 Jun 2024 21:20:24 -0500 Subject: [PATCH 50/53] Fix Fedora 40 as well --- distros/fedora/40/amd64-ks.cfg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/distros/fedora/40/amd64-ks.cfg b/distros/fedora/40/amd64-ks.cfg index 90ed9f8..9e19b81 100644 --- a/distros/fedora/40/amd64-ks.cfg +++ b/distros/fedora/40/amd64-ks.cfg @@ -1,5 +1,5 @@ user --name=vagrant --password=vagrant -rootpw vagrant +rootpw --plaintext vagrant text # don't use cmdline -- https://github.com/rhinstaller/anaconda/issues/931 lang en_US.UTF-8 @@ -7,7 +7,6 @@ keyboard us timezone --utc Etc/UTC selinux --enforcing -rootpw --lock --iscrypted locked firewall --disabled @@ -78,7 +77,6 @@ qemu-guest-agent -geolite2-city # Specific to vagrant -dnf-yum rsync fuse-sshfs %end From 297f3fc47361ad6a05d4cea655afc6dba870da0f Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 29 Jun 2024 21:31:19 -0500 Subject: [PATCH 51/53] Be more patient for Arch boot --- distros/archlinux/rolling/amd64.pkrvars.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distros/archlinux/rolling/amd64.pkrvars.hcl b/distros/archlinux/rolling/amd64.pkrvars.hcl index 6c0708c..395cab1 100644 --- a/distros/archlinux/rolling/amd64.pkrvars.hcl +++ b/distros/archlinux/rolling/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e net.ifnames=0 biosdevnames=0", - "", # Wait for system to come up? + "", # Wait for system to come up? "curl -o /tmp/install.sh http://{{ .HTTPIP }}:{{ .HTTPPort }}/install.sh", "", "/usr/bin/bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}" From 5f84855e5e85c61cc912ef40f33bbd85ddcb575b Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 29 Jun 2024 21:32:18 -0500 Subject: [PATCH 52/53] Also be more patient with NixOS --- distros/nixos/23.05/amd64.pkrvars.hcl.sh | 2 +- distros/nixos/23.11/amd64.pkrvars.hcl.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distros/nixos/23.05/amd64.pkrvars.hcl.sh b/distros/nixos/23.05/amd64.pkrvars.hcl.sh index b74a8da..0dbb92a 100755 --- a/distros/nixos/23.05/amd64.pkrvars.hcl.sh +++ b/distros/nixos/23.05/amd64.pkrvars.hcl.sh @@ -34,7 +34,7 @@ boot_command = [ "sudo bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}" ] http_directory = "distros/nixos/23.05/http/" -boot_wait = "60s" +boot_wait = "90s" EOF rm -r "${tmp}" diff --git a/distros/nixos/23.11/amd64.pkrvars.hcl.sh b/distros/nixos/23.11/amd64.pkrvars.hcl.sh index 73a5861..1078713 100755 --- a/distros/nixos/23.11/amd64.pkrvars.hcl.sh +++ b/distros/nixos/23.11/amd64.pkrvars.hcl.sh @@ -34,7 +34,7 @@ boot_command = [ "sudo bash /tmp/install.sh {{ .HTTPPort }} {{ .HTTPIP }}" ] http_directory = "distros/nixos/23.11/http/" -boot_wait = "60s" +boot_wait = "90s" EOF rm -r "${tmp}" From 4f4fac29c7e3c1686ac1e1a59e635b262c41181a Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 29 Jun 2024 22:03:55 -0500 Subject: [PATCH 53/53] Add serial output to QEmu Hopefully this will give a little more information on what is happening in the builds on QEmu servers. Tell builds to output to ttyS0 and then capture that and output to the console. --- distros/almalinux/9/amd64.pkrvars.hcl | 1 + distros/alpine/3.19.0/amd64.pkrvars.hcl | 2 +- distros/archlinux/rolling/amd64.pkrvars.hcl | 2 +- distros/centos-stream/9/amd64.pkrvars.hcl.sh | 1 + distros/centos/7/amd64.pkrvars.hcl | 1 + distros/debian/10/amd64.pkrvars.hcl | 1 + distros/debian/11/amd64.pkrvars.hcl | 1 + distros/debian/12/amd64.pkrvars.hcl | 1 + distros/fedora/39/amd64.pkrvars.hcl | 2 +- distros/fedora/40/amd64.pkrvars.hcl | 2 +- distros/fedora/rawhide/amd64.pkrvars.hcl.sh | 2 +- distros/opensuse/15.5/amd64.pkrvars.hcl | 1 + distros/opensuse/15.6/amd64.pkrvars.hcl | 1 + distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh | 1 + distros/rockylinux/9/amd64.pkrvars.hcl | 1 + distros/ubuntu/20.04/amd64.pkrvars.hcl | 2 +- distros/ubuntu/22.04/amd64.pkrvars.hcl | 2 +- distros/ubuntu/23.10/amd64.pkrvars.hcl | 2 +- sources/qemu.pkr.hcl | 3 ++- 19 files changed, 20 insertions(+), 9 deletions(-) diff --git a/distros/almalinux/9/amd64.pkrvars.hcl b/distros/almalinux/9/amd64.pkrvars.hcl index 83de82e..235dc15 100644 --- a/distros/almalinux/9/amd64.pkrvars.hcl +++ b/distros/almalinux/9/amd64.pkrvars.hcl @@ -8,6 +8,7 @@ boot_command = [ "e", "", " inst.text inst.ks=cdrom:/ks.cfg", + " console=ttyS0 inst.notmux", "x" ] cd_files = [ "distros/almalinux/9/disc/*" ] diff --git a/distros/alpine/3.19.0/amd64.pkrvars.hcl b/distros/alpine/3.19.0/amd64.pkrvars.hcl index 47936c3..894f4fd 100644 --- a/distros/alpine/3.19.0/amd64.pkrvars.hcl +++ b/distros/alpine/3.19.0/amd64.pkrvars.hcl @@ -16,4 +16,4 @@ boot_command = [ ] http_directory = "distros/alpine/3.19.0/http/" boot_wait = "20s" -shutdown_command = "sudo poweroff" \ No newline at end of file +shutdown_command = "sudo poweroff" diff --git a/distros/archlinux/rolling/amd64.pkrvars.hcl b/distros/archlinux/rolling/amd64.pkrvars.hcl index 395cab1..60f9fdb 100644 --- a/distros/archlinux/rolling/amd64.pkrvars.hcl +++ b/distros/archlinux/rolling/amd64.pkrvars.hcl @@ -5,7 +5,7 @@ iso = { checksum = "52aea58f88c9a80afe64f0536da868251ef4878de5a5e0227fcada9f132bd7ab" } boot_command = [ - "e net.ifnames=0 biosdevnames=0", + "e net.ifnames=0 biosdevnames=0 console=ttyS0", "", # Wait for system to come up? "curl -o /tmp/install.sh http://{{ .HTTPIP }}:{{ .HTTPPort }}/install.sh", "", diff --git a/distros/centos-stream/9/amd64.pkrvars.hcl.sh b/distros/centos-stream/9/amd64.pkrvars.hcl.sh index f0fa813..2e5c192 100755 --- a/distros/centos-stream/9/amd64.pkrvars.hcl.sh +++ b/distros/centos-stream/9/amd64.pkrvars.hcl.sh @@ -24,6 +24,7 @@ boot_command = [ "e", "", " inst.text inst.ks=cdrom:/ks.cfg", + " inst.notmux console=ttyS0", "x" ] cd_files = [ "distros/centos-stream/9/disc/*" ] diff --git a/distros/centos/7/amd64.pkrvars.hcl b/distros/centos/7/amd64.pkrvars.hcl index b553bb1..1926391 100644 --- a/distros/centos/7/amd64.pkrvars.hcl +++ b/distros/centos/7/amd64.pkrvars.hcl @@ -8,6 +8,7 @@ boot_command = [ "e", "", " inst.ks=cdrom:/ks.cfg", + " inst.notmux console=ttyS0", "x" ] cd_files = ["distros/centos/7/disc/*"] diff --git a/distros/debian/10/amd64.pkrvars.hcl b/distros/debian/10/amd64.pkrvars.hcl index 7a1aeb1..94d2556 100644 --- a/distros/debian/10/amd64.pkrvars.hcl +++ b/distros/debian/10/amd64.pkrvars.hcl @@ -16,6 +16,7 @@ boot_command = [ "debconf/frontend=noninteractive ", "hostname={{.Name}} ", "domain=vagrant ", + "console=ttyS0 ", "" ] http_directory = "distros/debian/10/http/" diff --git a/distros/debian/11/amd64.pkrvars.hcl b/distros/debian/11/amd64.pkrvars.hcl index 795b6cb..e93396e 100644 --- a/distros/debian/11/amd64.pkrvars.hcl +++ b/distros/debian/11/amd64.pkrvars.hcl @@ -16,6 +16,7 @@ boot_command = [ "debconf/frontend=noninteractive ", "hostname={{.Name}} ", "domain=vagrant ", + "console=ttyS0 ", "" ] http_directory = "distros/debian/11/http/" diff --git a/distros/debian/12/amd64.pkrvars.hcl b/distros/debian/12/amd64.pkrvars.hcl index d65e049..3278968 100644 --- a/distros/debian/12/amd64.pkrvars.hcl +++ b/distros/debian/12/amd64.pkrvars.hcl @@ -16,6 +16,7 @@ boot_command = [ "debconf/frontend=noninteractive ", "hostname={{.Name}} ", "domain=vagrant ", + "console=ttyS0 ", "" ] http_directory = "distros/debian/12/http/" diff --git a/distros/fedora/39/amd64.pkrvars.hcl b/distros/fedora/39/amd64.pkrvars.hcl index 2a25ab1..6c3821c 100644 --- a/distros/fedora/39/amd64.pkrvars.hcl +++ b/distros/fedora/39/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e ", - "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0" + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0" ] cd_files = [ "distros/fedora/39/*.cfg" diff --git a/distros/fedora/40/amd64.pkrvars.hcl b/distros/fedora/40/amd64.pkrvars.hcl index cde6fce..76fd884 100644 --- a/distros/fedora/40/amd64.pkrvars.hcl +++ b/distros/fedora/40/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e ", - "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0" + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0" ] cd_files = [ "distros/fedora/40/*.cfg" diff --git a/distros/fedora/rawhide/amd64.pkrvars.hcl.sh b/distros/fedora/rawhide/amd64.pkrvars.hcl.sh index fccdaab..fa06d83 100755 --- a/distros/fedora/rawhide/amd64.pkrvars.hcl.sh +++ b/distros/fedora/rawhide/amd64.pkrvars.hcl.sh @@ -35,7 +35,7 @@ iso = { } boot_command = [ "e", - "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg console=tty0 console=ttyS0" + "net.ifnames=0 biosdevnames=0 inst.ks=cdrom:/amd64-ks.cfg inst.notmux console=ttyS0" ] cd_files = [ "distros/fedora/rawhide/*.cfg" diff --git a/distros/opensuse/15.5/amd64.pkrvars.hcl b/distros/opensuse/15.5/amd64.pkrvars.hcl index 8b80ad3..ce2b718 100644 --- a/distros/opensuse/15.5/amd64.pkrvars.hcl +++ b/distros/opensuse/15.5/amd64.pkrvars.hcl @@ -9,6 +9,7 @@ boot_command = [ "", "", " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1", + " console=ttyS0", "x" ] http_directory = "distros/opensuse/15.5/http/" diff --git a/distros/opensuse/15.6/amd64.pkrvars.hcl b/distros/opensuse/15.6/amd64.pkrvars.hcl index 64a74be..8e158a1 100644 --- a/distros/opensuse/15.6/amd64.pkrvars.hcl +++ b/distros/opensuse/15.6/amd64.pkrvars.hcl @@ -9,6 +9,7 @@ boot_command = [ "", "", " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1", + " console=ttyS0", "x" ] http_directory = "distros/opensuse/15.6/http/" diff --git a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh index 191c91b..9a6fe9c 100755 --- a/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh +++ b/distros/opensuse/tumbleweed/amd64.pkrvars.hcl.sh @@ -26,6 +26,7 @@ boot_command = [ "e", "", " autoyast=http://{{ .HTTPIP }}:{{ .HTTPPort }}/autoinst.xml textmode=1", + " console=ttyS0", "x" ] http_directory = "distros/opensuse/tumbleweed/http/" diff --git a/distros/rockylinux/9/amd64.pkrvars.hcl b/distros/rockylinux/9/amd64.pkrvars.hcl index be8f547..12da5e9 100644 --- a/distros/rockylinux/9/amd64.pkrvars.hcl +++ b/distros/rockylinux/9/amd64.pkrvars.hcl @@ -8,6 +8,7 @@ boot_command = [ "e", "", " inst.text inst.ks=cdrom:/ks.cfg", + " inst.notmux console=ttyS0", "x" ] cd_files = [ "distros/rockylinux/9/disc/*" ] diff --git a/distros/ubuntu/20.04/amd64.pkrvars.hcl b/distros/ubuntu/20.04/amd64.pkrvars.hcl index 5c49cad..a95048e 100644 --- a/distros/ubuntu/20.04/amd64.pkrvars.hcl +++ b/distros/ubuntu/20.04/amd64.pkrvars.hcl @@ -7,7 +7,7 @@ iso = { boot_wait = "3s" boot_command = [ "e", - "console=tty0 ", + "console=ttyS0 ", "fsck.mode=skip ", "locale=en_US ", "auto=true ", diff --git a/distros/ubuntu/22.04/amd64.pkrvars.hcl b/distros/ubuntu/22.04/amd64.pkrvars.hcl index 96f325b..51baf9b 100644 --- a/distros/ubuntu/22.04/amd64.pkrvars.hcl +++ b/distros/ubuntu/22.04/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e", - "console=tty0 ", + "console=ttyS0 ", "fsck.mode=skip ", "locale=en_US ", "auto=true ", diff --git a/distros/ubuntu/23.10/amd64.pkrvars.hcl b/distros/ubuntu/23.10/amd64.pkrvars.hcl index 3c8832a..59d3759 100644 --- a/distros/ubuntu/23.10/amd64.pkrvars.hcl +++ b/distros/ubuntu/23.10/amd64.pkrvars.hcl @@ -6,7 +6,7 @@ iso = { } boot_command = [ "e", - "console=tty0 ", + "console=ttyS0 ", "fsck.mode=skip ", "locale=en_US ", "auto=true ", diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index ffcfaa9..d1b82ee 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -31,10 +31,11 @@ source "qemu" "amd64" { efi_firmware_vars = "${var.efi_dir}OVMF_VARS.fd" qemuargs = [ - ["-chardev", "stdio,id=char0,logfile=serial-output-qemu-${var.arch}-${var.distro}-${var.version},signal=off"], + #["-chardev", "stdio,id=char0,logfile=serial-output-qemu-${var.arch}-${var.distro}-${var.version},signal=off"], ["-chardev", "socket,path=${local.name}.qga.sock,server=on,wait=off,id=qga0"], ["-device", "virtio-serial"], ["-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"], + ["-serial", "stdio"] ] accelerator = var.qemu_accelerator