diff --git a/.github/actions/install_packer/action.yml b/.github/actions/install_packer/action.yml index 6fd14fa..3664593 100644 --- a/.github/actions/install_packer/action.yml +++ b/.github/actions/install_packer/action.yml @@ -17,8 +17,10 @@ runs: if: runner.os == 'Linux' run: | set -exo pipefail + # Install virtualbox sudo apt-get update - sudo apt-get install -y unzip python3-{virtualenv,yaml} cloud-utils qemu-system-{x86,ppc} + # Install QEmu, etc + sudo apt-get install -y unzip python3-{virtualenv,yaml} cloud-utils xorriso qemu-system-{x86,ppc} # Install packer.io curl -O https://releases.hashicorp.com/packer/${{ inputs.version }}/packer_${{ inputs.version }}_linux_amd64.zip unzip -o packer_${{ inputs.version }}_linux_amd64.zip -d /usr/local/bin @@ -31,6 +33,5 @@ runs: if: runner.os == 'macOS' run: | set -exo pipefail - brew install qemu - brew install pup + brew install pup dvdrtools xorriso coreutils gsed # Packer is already installed diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1930209..35ad794 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,12 +25,15 @@ jobs: build: name: Build box - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.builder == 'virtualbox-iso.x86_64' && 'macOS-12' || 'ubuntu-22.04' }} needs: - collect strategy: fail-fast: false matrix: + builder: + - virtualbox-iso.x86_64 + - qemu.x86_64 distro: ${{ fromJson( needs.collect.outputs.distros ) }} needs_timeout: - distros/ubuntu-20.04-x86_64.pkrvars.hcl @@ -39,28 +42,35 @@ jobs: uses: actions/checkout@v3 with: submodules: recursive - - uses: ./.github/actions/setup + - name: Setup system dependnecies + uses: ./.github/actions/setup with: ansible: true - - uses: ./.github/actions/install_packer - - uses: greg-hellings/make-space@main - - name: modify build timeouts, if necessary + - name: Install Packer and related tools + uses: ./.github/actions/install_packer + - name: Clear disk space + uses: greg-hellings/make-space@main + - name: Modify build timeouts, if necessary if: contains(matrix.needs_timeout, matrix.distro) shell: bash run: ./modify_timeout.sh "${{ matrix.distro }}" - name: Run generator, if necessary if: endsWith(matrix.distro, '.sh') run: "${{ matrix.distro }}" - - name: try build + - name: Build shell: bash run: | set -e # Strip .sh if it is a dynamic one file="$(printf "${{ matrix.distro }}" | sed -E -e 's/^(.*).sh$/\1/')" # Add a retry in case something bogus like a network failure happens - until packer build -var-file="${file}" -var "qemu_accelerator=tcg" ${{ github.event_name != 'release' && '-except=upload' || '' }} sources; do - sleep 1 - done + packer build -only="${{ matrix.builder }}" -var-file="${file}" -var "qemu_accelerator=tcg" ${{ github.event_name != 'release' && '-except=upload' || '' }} sources env: VAGRANT_CLOUD_TOKEN: ${{ secrets.VAGRANT_CLOUD_TOKEN }} - PACKER_LOG: ${{ contains(matrix.needs_timeout, matrix.distro) && '1' || '1' }} + PACKER_LOG: ${{ contains(matrix.needs_timeout, matrix.distro) && '0' || '0' }} + - name: Upload artifact + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: serial-output + path: serial-output* diff --git a/.gitignore b/.gitignore index 795f53b..a1922e2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ output_* *.box *.iso .vagrant +serial-output +grub.cfg +isolinux.cfg # Generated distros distros/fedora-rawhide-*.hcl diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 1bec74c..a3491fd 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -1,18 +1,22 @@ - name: install guest additions hosts: default pre_tasks: - - name: update all the things + - name: Update all the things become: true ansible.builtin.package: # noqa package-latest name: "*" state: latest when: ansible_facts.pkg_mgr not in ['atomic_container'] - - name: update atomic things + - name: Update atomic things become: true ansible.builtin.command: rpm-ostree upgrade when: ansible_facts.pkg_mgr in ['atomic_container'] changed_when: false + + - name: Restart the machine + become: true + ansible.builtin.reboot: roles: - role: virtualbox when: packer_builder_type == 'virtualbox-iso' diff --git a/ansible/roles/virtualbox/molecule/shared/playbook.yml b/ansible/roles/virtualbox/molecule/shared/playbook.yml new file mode 100644 index 0000000..e032241 --- /dev/null +++ b/ansible/roles/virtualbox/molecule/shared/playbook.yml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - virtualbox diff --git a/ansible/roles/virtualbox/molecule/ubuntu18/molecule.yml b/ansible/roles/virtualbox/molecule/ubuntu18/molecule.yml new file mode 100644 index 0000000..7193175 --- /dev/null +++ b/ansible/roles/virtualbox/molecule/ubuntu18/molecule.yml @@ -0,0 +1,8 @@ +driver: + name: containers + provider: + name: libvirt +platforms: + - name: test-ubuntu18-init + image: ubuntu:18.04 + box: generic/ubuntu1804 diff --git a/ansible/roles/virtualbox/molecule/ubuntu20/molecule.yml b/ansible/roles/virtualbox/molecule/ubuntu20/molecule.yml new file mode 100644 index 0000000..f570cba --- /dev/null +++ b/ansible/roles/virtualbox/molecule/ubuntu20/molecule.yml @@ -0,0 +1,8 @@ +driver: + name: containers + provider: + name: libvirt +platforms: + - name: test-ubuntu20-init + image: ubuntu:20.04 + box: generic/ubuntu2004 diff --git a/ansible/roles/virtualbox/tasks/main.yml b/ansible/roles/virtualbox/tasks/main.yml index c026bd3..4f93540 100644 --- a/ansible/roles/virtualbox/tasks/main.yml +++ b/ansible/roles/virtualbox/tasks/main.yml @@ -1,10 +1,23 @@ -- name: install build deps for virtualbox addons +- name: Load version information + ansible.builtin.include_vars: "{{ item }}" + loop: + - "{{ role_path }}/vars/{{ ansible_facts['os_family'] }}.yml" + - "{{ role_path }}/vars/{{ ansible_facts['distribution'] }}.yml" + - "{{ role_path }}/vars/{{ ansible_facts['distribution'] }}_{{ ansible_facts['distribution_major_version'] }}.yml" + - "{{ role_path }}/vars/{{ ansible_facts['distribution'] }}_{{ ansible_facts['distribution_version'] }}.yml" + when: item is file + +- name: Install build deps for virtualbox addons become: true ansible.builtin.package: name: "{{ virtualbox_packages }}" state: present + update_cache: true + register: _pkg_install + retries: 3 + until: _pkg_install is success -- name: start services +- name: Start services become: true ansible.builtin.service: name: "{{ item }}" @@ -12,16 +25,21 @@ enabled: true loop: "{{ virtualbox_services }}" -- name: mount iso file - become: true - ansible.builtin.command: "{{ item }}" - changed_when: false - loop: - - mount -o loop,ro {{ ansible_user_dir }}/VBoxGuestAdditions.iso /mnt - - /mnt/VBoxLinuxAdditions.run - - umount /mnt +- when: virtualbox_from_iso + block: + - name: Mount iso file + become: true + ansible.builtin.command: "{{ item }}" + changed_when: false + failed_when: _cmd.rc not in [0, 2] + register: _cmd + loop: + - mount -o loop,ro {{ virtualbox_iso_location }} /mnt + - /mnt/VBoxLinuxAdditions.run --nox11 + - umount /mnt -- name: remove iso - ansible.builtin.file: - name: "{{ ansible_user_dir }}/VBoxGuestAdditions.iso" +- name: Remove packages + become: true + ansible.builtin.package: + name: "{{ virtualbox_remove_packages | default([]) }}" state: absent diff --git a/ansible/roles/virtualbox/vars/CentOS.yml b/ansible/roles/virtualbox/vars/CentOS.yml index a65ba56..22b19d1 100644 --- a/ansible/roles/virtualbox/vars/CentOS.yml +++ b/ansible/roles/virtualbox/vars/CentOS.yml @@ -6,3 +6,4 @@ virtualbox_packages: - perl virtualbox_services: - dkms +virtualbox_from_iso: true diff --git a/ansible/roles/virtualbox/vars/Fedora.yml b/ansible/roles/virtualbox/vars/Fedora.yml index 6ace794..3b597df 100644 --- a/ansible/roles/virtualbox/vars/Fedora.yml +++ b/ansible/roles/virtualbox/vars/Fedora.yml @@ -1,7 +1,16 @@ virtualbox_packages: + - kernel + - kernel-devel + - kernel-headers + - gcc + - make + - perl + - elfutils-libelf-devel + - tar - bzip2 - dkms - - kernel-devel - - make virtualbox_services: - dkms +virtualbox_iso_location: "{{ ansible_user_dir }}/VBoxGuestAdditions.iso" +virtualbox_from_iso: true +virtualbox_pre_commands: [] diff --git a/ansible/roles/virtualbox/vars/Ubuntu.yml b/ansible/roles/virtualbox/vars/Ubuntu.yml index eb6f954..5dab71c 100644 --- a/ansible/roles/virtualbox/vars/Ubuntu.yml +++ b/ansible/roles/virtualbox/vars/Ubuntu.yml @@ -1,5 +1,5 @@ virtualbox_packages: - dkms - - make -virtualbox_services: - - dkms + - virtualbox-guest-dkms +virtualbox_services: [] +virtualbox_from_iso: false diff --git a/ansible/roles/virtualbox/vars/Ubuntu_22.04.yml b/ansible/roles/virtualbox/vars/Ubuntu_22.04.yml new file mode 100644 index 0000000..41378a3 --- /dev/null +++ b/ansible/roles/virtualbox/vars/Ubuntu_22.04.yml @@ -0,0 +1,9 @@ +virtualbox_packages: + - virtualbox-guest-utils + - bzip2 + - tar + - build-essential +virtualbox_services: [] +virtualbox_from_iso: true +virtualbox_iso_location: /home/vagrant/VBoxGuestAdditions.iso +virtualbox_remove_packages: [] diff --git a/distros/fedora-36-x86_64.pkrvars.hcl b/distros/fedora-36-x86_64.pkrvars.hcl index 0408337..f48455f 100644 --- a/distros/fedora-36-x86_64.pkrvars.hcl +++ b/distros/fedora-36-x86_64.pkrvars.hcl @@ -6,5 +6,5 @@ iso = { } boot_command = [ "", - "net.ifnames=0 biosdevnames=0 inst.ks=http://{{.HTTPIP}}:{{.HTTPPort}}/fedora/36-x86_64.ks console=ttyS0 " + "net.ifnames=0 biosdevnames=0 inst.ks=http://{{.HTTPIP}}:{{.HTTPPort}}/fedora/36-x86_64.ks console=ttyS0 console=tty0" ] diff --git a/distros/fedora-rawhide-x86_64.pkrvars.hcl.sh b/distros/fedora-rawhide-x86_64.pkrvars.hcl.sh index 3244172..fb8bee7 100755 --- a/distros/fedora-rawhide-x86_64.pkrvars.hcl.sh +++ b/distros/fedora-rawhide-x86_64.pkrvars.hcl.sh @@ -35,7 +35,7 @@ iso = { } boot_command = [ "", - "net.ifnames=0 biosdevnames=0 inst.ks=http://{{.HTTPIP}}:{{.HTTPPort}}/fedora/rawhide-x86_64.ks console=ttyS0 " + "edd=off net.ifnames=0 biosdevnames=0 inst.ks=http://{{.HTTPIP}}:{{.HTTPPort}}/fedora/rawhide-x86_64.ks console=ttyS0 " ] EOF diff --git a/distros/ubuntu-18.04-x86_64.pkrvars.hcl b/distros/ubuntu-18.04-x86_64.pkrvars.hcl index 07c26c9..c4694a9 100644 --- a/distros/ubuntu-18.04-x86_64.pkrvars.hcl +++ b/distros/ubuntu-18.04-x86_64.pkrvars.hcl @@ -13,6 +13,7 @@ boot_command = [ "DEBIAN_FRONTEND=noninteractive ", "preseed/url=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/18.04/preseed.cfg ", "fsck.mode=skip ", - "console=ttyS0", + "console=ttyS0 ", + "console=tty0 ", "" ] diff --git a/distros/ubuntu-20.04-x86_64.pkrvars.hcl b/distros/ubuntu-20.04-x86_64.pkrvars.hcl index 6cd60a1..5f22b8c 100644 --- a/distros/ubuntu-20.04-x86_64.pkrvars.hcl +++ b/distros/ubuntu-20.04-x86_64.pkrvars.hcl @@ -8,6 +8,7 @@ boot_wait = "3s" boot_command = [ "", "console=ttyS0 ", + "console=tty0 ", "fsck.mode=skip ", "autoinstall ds=nocloud-net;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/20.04/" ] diff --git a/http/fedora/34-x86_64.ks b/http/fedora/34-x86_64.ks index 970563f..b9ebbfc 100644 --- a/http/fedora/34-x86_64.ks +++ b/http/fedora/34-x86_64.ks @@ -8,4 +8,8 @@ ks=$(printf "${cmdline}" | sed -E -e 's/.*inst\.ks=([^ ]*).*/\1/') mkdir -p /tmp/packer_ks cd /tmp/packer_ks wget -m -nH "$(dirname "${ks}")" + +if [ ! -e /dev/vda ]; then + ln -s /dev/sda /dev/vda +fi %end diff --git a/http/fedora/35-x86_64.ks b/http/fedora/35-x86_64.ks index ed17021..1d6cd3d 100644 --- a/http/fedora/35-x86_64.ks +++ b/http/fedora/35-x86_64.ks @@ -16,4 +16,8 @@ ks=$(printf "${cmdline}" | sed -E -e 's/.*inst\.ks=([^ ]*).*/\1/') mkdir -p /tmp/packer_ks cd /tmp/packer_ks wget -m -nH "$(dirname "${ks}")" + +if [ ! -e /dev/vda ]; then + ln -s /dev/sda /dev/vda +fi %end diff --git a/http/fedora/36-x86_64.ks b/http/fedora/36-x86_64.ks index d7a3057..2911c86 100644 --- a/http/fedora/36-x86_64.ks +++ b/http/fedora/36-x86_64.ks @@ -16,4 +16,8 @@ ks=$(printf "${cmdline}" | sed -E -e 's/.*inst\.ks=([^ ]*).*/\1/') mkdir -p /tmp/packer_ks cd /tmp/packer_ks wget -m -nH "$(dirname "${ks}")" + +if [ ! -e /dev/vda ]; then + ln -s /dev/sda /dev/vda +fi %end diff --git a/http/fedora/f36 b/http/fedora/f36 index a69afdb..dfbad4b 160000 --- a/http/fedora/f36 +++ b/http/fedora/f36 @@ -1 +1 @@ -Subproject commit a69afdb7e04c06cde624a9de2444d755e95ee8b4 +Subproject commit dfbad4bdf5f5b9c6a93b3b192bfe91580ac08915 diff --git a/http/fedora/rawhide b/http/fedora/rawhide index 1fcebbb..1841b36 160000 --- a/http/fedora/rawhide +++ b/http/fedora/rawhide @@ -1 +1 @@ -Subproject commit 1fcebbbe2277184f2bc5963180434fb559d59053 +Subproject commit 1841b368cef9b1c6226644301ed5c3893a363422 diff --git a/http/fedora/rawhide-x86_64.ks b/http/fedora/rawhide-x86_64.ks index 80fff9e..085f975 100644 --- a/http/fedora/rawhide-x86_64.ks +++ b/http/fedora/rawhide-x86_64.ks @@ -16,4 +16,8 @@ ks=$(printf "${cmdline}" | sed -E -e 's/.*inst\.ks=([^ ]*).*/\1/') mkdir -p /tmp/packer_ks cd /tmp/packer_ks wget -m -nH "$(dirname "${ks}")" + +if [ ! -e /dev/vda ]; then + ln -s /dev/sda /dev/vda +fi %end diff --git a/http/ubuntu/20.04/user-data b/http/ubuntu/20.04/user-data index 04baedc..bb48502 100644 --- a/http/ubuntu/20.04/user-data +++ b/http/ubuntu/20.04/user-data @@ -2,12 +2,22 @@ # vim: set ft=yaml: autoinstall: version: 1 + apt: + preserve_sources_list: false + primary: + - arches: [i386, amd64] + uri: http://archive.ubuntu.com/ubuntu + - arches: [default] + uri: http://ports.ubuntu.com/ubuntu-ports + geoip: false network: # Should disable networking, workaround for unattended-updates failing in install version: 2 refresh-installer: update: no keyboard: layout: us + variant: "" + toggle: null storage: config: - id: drive @@ -25,7 +35,7 @@ autoinstall: - id: boot-partition type: partition device: drive - size: 200M + size: 1G name: boot - id: swap-partition type: partition diff --git a/modify_timeout.sh b/modify_timeout.sh index 4a07f8e..0407bb2 100755 --- a/modify_timeout.sh +++ b/modify_timeout.sh @@ -7,39 +7,32 @@ distro="${1}" url="$(cat "${distro}" | grep url | cut -f2 -d'"')" curl -C - -O "${url}" -# Mount and copy +# Mount and extract file=$(printf "${url}" | sed -E -e 's#.*/(.*)$#\1#') -old=$(mktemp -d) -new=$(mktemp -d) -sudo mount -t iso9660 -o loop "${file}" "${old}" -{ - cd "${old}" - tar -cvf - . | (cd "${new}" && tar -xf -) - cd - -} +work="$(mktemp -d)" +xorriso -osirrox on -indev "${file}" -extract / "${work}" # Modify timeout -chmod +w "${new}/boot/grub" "${new}/boot/grub/grub.cfg" -chmod +w "${new}/isolinux" "${new}/isolinux/isolinux.cfg" -sed -i -e 's/set timeout=.*/set timeout=30/' "${new}/boot/grub/grub.cfg" -sed -i -e 's/timeout.*/timeout 300/' "${new}/isolinux/isolinux.cfg" # Uses 10ths of a second -chmod -w "${new}/boot/grub" "${new}/boot/grub/grub.cfg" -chmod -w "${new}/isolinux" "${new}/isolinux/isolinux.cfg" -cat "${new}/boot/grub/grub.cfg" -cat "${new}/isolinux/isolinux.cfg" +chmod -R +w "${work}" +sed -i -e 's/set timeout=.*/set timeout=30/' "${work}/boot/grub/grub.cfg" +sed -i -e 's/timeout.*/timeout 300/' "${work}/isolinux/isolinux.cfg" # Uses 10ths of a second +chmod -R -w "${work}" # Create new ISO file -sudo umount "${old}" rm "${file}" -chmod +w "${new}/isolinux/isolinux.bin" -mkisofs -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -J -R -V "Boot" -r -iso-level 4 -chrp-boot -o "${file}" "${new}" +chmod +w "${work}/ubuntu" "${work}/isolinux/isolinux.bin" +mkisofs -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -J -R -V "Boot" -r -iso-level 3 -chrp-boot -o "${file}" "${work}" new_sum=$(sha256sum "${file}" | cut -f1 -d' ') # Update distro file -sed -i "${distro}" -e "s/checksum = \".*\"/checksum = \"${new_sum}\"/" -sed -i "${distro}" -e "s/url = \".*\"/url = \"${file}\"/" -sed -i "${distro}" -e "s/boot_wait = \".*\"/boot_wait = \"15s\"/" +if [ "$(uname)" == "Darwin" ]; then + SED=$(which gsed) +else + SED=$(which sed) +fi +"${SED}" -i -e "s/checksum = \".*\"/checksum = \"${new_sum}\"/" "${distro}" +"${SED}" -i -e "s/url = \".*\"/url = \"${file}\"/" "${distro}" +"${SED}" -i -e "s/boot_wait = \".*\"/boot_wait = \"15s\"/" "${distro}" # Cleanup remaining directories -rm -r "${old}" -#sudo rm -r "${new}" +rm -r "${work}" || true diff --git a/scripts/minimize.sh b/scripts/minimize.sh index f07bba9..4e7d753 100644 --- a/scripts/minimize.sh +++ b/scripts/minimize.sh @@ -6,12 +6,12 @@ set -x rm -rf /home/vagrant/.ansible sudo rm -rf /root/.ansible -#if [[ "${PACKER_BUILD_NAME}" != *"silverblue"* ]]; then +if [ ! -e /etc/fedora-release ]; then sudo dd if=/dev/zero of=/var/EMPTY bs=1M || : sync sudo rm /var/EMPTY sync -#fi +fi # In CentOS 7, blkid returns duplicate devices swap_device_uuid=`sudo /sbin/blkid -t TYPE=swap -o value -s UUID | uniq` diff --git a/sources/build.pkr.hcl b/sources/build.pkr.hcl index 1f429e7..72fed16 100644 --- a/sources/build.pkr.hcl +++ b/sources/build.pkr.hcl @@ -1,6 +1,7 @@ build { sources = [ "source.qemu.x86_64", + "source.virtualbox-iso.x86_64" ] provisioner "ansible" { diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl index d14a32c..270fe93 100644 --- a/sources/qemu.pkr.hcl +++ b/sources/qemu.pkr.hcl @@ -24,7 +24,10 @@ source "qemu" "x86_64" { disk_cache = "writeback" format = "qcow2" - qemuargs = [["-serial", "stdio"]] + qemuargs = [ + ["-chardev", "stdio,id=char0,logfile=serial-output-qemu-${var.arch}-${var.distro}-${var.version},signal=off"], + ["-serial", "chardev:char0"] + ] http_directory = "http" accelerator = var.qemu_accelerator } diff --git a/sources/virtualbox-iso.pkr.hcl b/sources/virtualbox-iso.pkr.hcl index a52209a..36087a3 100644 --- a/sources/virtualbox-iso.pkr.hcl +++ b/sources/virtualbox-iso.pkr.hcl @@ -18,6 +18,12 @@ source "virtualbox-iso" "x86_64" { memory = local.memory headless = var.headless guest_os_type = "${var.distro}_64" + guest_additions_url = "http://download.virtualbox.org/virtualbox/6.1.34/VBoxGuestAdditions_6.1.34.iso" + guest_additions_sha256 = "88f86fa0e6970b6a7c80d714b7a91a8c425ff8ef53a3e73fc80781191a87257b" + vboxmanage = [ + ["modifyvm", "{{ .Name }}", "--uart1", "0x3F8", "4"], + ["modifyvm", "{{ .Name }}", "--uartmode1", "file", "serial-output-virtualbox-${var.arch}-${var.distro}-${var.version}"] + ] http_directory = "http" } diff --git a/test.sh b/test.sh index 7b8854b..d30e590 100755 --- a/test.sh +++ b/test.sh @@ -3,8 +3,10 @@ set -exo pipefail build="${1}" +provider=virtualbox-iso.x86_64 +#provider=qemu.x86_64 -packer build -var headless=false -var-file distros/${build}.pkrvars.hcl -except=upload sources/ +packer build -on-error=ask -var headless=false -var-file distros/${build}.pkrvars.hcl -except=upload -only=${provider} sources/ vagrant box add -f --name test ${build}.box vagrant up vagrant ssh -c "ls -la ~/"