Do some better linting (#13)

* Do some better linting * Update names, Packer version, and more * Do not lint on push * More linting and scripting * Cleanup shellcheck lint * Add Packer format check * Fix Packer formatting * Limit to only one * Remove generated HCL files * Only lint checked files * Use pipes instead * Improve attempt to build Actually go back to attempting to build boxes on GitHub. Maybe it will work better with this structure * Change to using environment files * Initialize plugins, first * More verbose usage of the file * Do the output properly this time * PACKER_LOG set to 1 * No need for python3 anymore * Limit to only one result * Choose a specific CPU model * Init separately * Update actions version * Add version outputting * Correct actions version * Try to execute qemu properly * Fix CPU model name * Do not over-subscribe GitHub runners * Try type instead of command to guard runs * Do not write to non-existent folder * Cache packer plugins * Use ~ instead of /home/greg * Try the proper path * Move to intelligent matrix * Set global env path * Add lots of waits for VBoxManage on hosted runners * Add more waits * Add more waits
2023-12-06 10:21:32 -06:00
parent 86dfe113a0
commit 634aae8d75
55 changed files with 536 additions and 495 deletions
@@ -8,72 +8,113 @@ name: Build the boxes
      - prereleased
  pull_request:

+env:
+  PACKER_PLUGIN_PATH: "~/.config/packer/plugins"
+
 jobs:
  collect:
    name: Discover distros
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
    outputs:
      distros: ${{ steps.distros.outputs.distros }}
      build: ${{ steps.time.outputs.time }}
    steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3
+          cache: pip
+          cache-dependency-path: .github/workflows/build.yml
+
      - name: Discover boxes
        id: distros
        run: |-
-          set -eo pipefail
-          distros=$(ls distros/*.hcl{,.sh} | jq -cRs 'split("\n") | [ .[] | if length > 0 then . else empty end ]')
-          echo "::set-output name=distros::${distros}"
+          set -exo pipefail
+          pip install xonsh PyYAML
+          echo "distros=$(xonsh build.xsh -a --list)" >> "$GITHUB_OUTPUT"
+
      - name: Find timestamp for versioning
        id: time
-        uses: nanzm/get-time-action@v1.1
+        uses: nanzm/get-time-action@v2.0
        with:
          format: 'YYYY.MM.DD.HH'

  build:
    name: Build box
-    runs-on: ${{ matrix.builder == 'virtualbox-iso.amd64' && 'macOS-12' || 'ubuntu-22.04' }}
+    runs-on: ${{ matrix.runner }}
    needs:
      - collect
    strategy:
      fail-fast: false
-      matrix:
-        builder:
-          - virtualbox-iso.amd64
-          - qemu.amd64
-        distro: ${{ fromJson( needs.collect.outputs.distros ) }}
-        needs_timeout:
-          - distros/ubuntu-20.04-x86_64.pkrvars.hcl
+      matrix: ${{ fromJson( needs.collect.outputs.distros ) }}
    steps:
      - name: Checkout project
        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-      - name: Setup system dependnecies
-        uses: ./.github/actions/setup
-        with:
-          ansible: true
+
      - name: Install Packer and related tools
        uses: ./.github/actions/install_packer
+
      - name: Clear disk space
        uses: greg-hellings/make-space@main
-      - name: Modify build timeouts, if necessary
-        if: contains(matrix.needs_timeout, matrix.distro)
+
+      - name: Output tool versions
        shell: bash
-        run: ./modify_timeout.sh "${{ matrix.distro }}"
+        run: |
+          set -x
+          packer -version
+          if $(type qemu-system-x86_64); then
+            qemu-system-x86_64 -version
+          fi
+          if $(type VBoxManage); then
+            VBoxManage --version;
+          fi
+
      - name: Run generator, if necessary
-        if: endsWith(matrix.distro, '.sh')
-        run: "${{ matrix.distro }}"
+        shell: bash
+        run: |
+          if [ -f "${{ matrix.distro }}.sh" ]; then
+            "./${{ matrix.distro }}.sh"
+          fi
+
+      - name: Restore Packer plugin cache
+        uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: ~/.config/packer/plugins
+          key: ${{ runner.os }}-${{ hashFiles('sources/build.pkr.hcl') }}
+
+      - name: Install Packer plugins
+        shell: bash
+        run: packer init sources
+
+      - name: Cache Packer plugins
+        uses: actions/cache/save@v3
+        if: always()
+        with:
+          path: ~/.config/packer/plugins
+          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
+
      - name: Build
        shell: bash
        run: |
          set -e
-          # Strip .sh if it is a dynamic one
-          file="$(printf "${{ matrix.distro }}" | sed -E -e 's/^(.*).sh$/\1/')"
          # Add a retry in case something bogus like a network failure happens
-          packer build -only="${{ matrix.builder }}" -var-file="${file}" -var "qemu_accelerator=tcg" -var "build=${{ needs.collect.outputs.build }}" ${{ github.event_name != 'release' && '-except=upload' || '' }} sources
+          packer build -only="${{ matrix.builder }}" \
+                       -var-file="${{ matrix.distro }}" \
+                       -var "qemu_accelerator=tcg" \
+                       -var "build=${{ needs.collect.outputs.build }}" \
+                       -var "cpu_model=Skylake-Client" \
+                       -var "cpus=2" \
+                       -var "memory=4096" \
+                       ${{ github.event_name != 'release' && '-except=upload' || '' }} \
+                       sources
        env:
          VAGRANT_CLOUD_TOKEN: ${{ secrets.VAGRANT_CLOUD_TOKEN }}
-          PACKER_LOG: ${{ contains(matrix.needs_timeout, matrix.distro) && '0' || '0' }}
+          PACKER_LOG: 1 #${{ contains(matrix.needs_timeout, matrix.distro) && '0' || '0' }}
+
      - name: Upload artifact
        if: ${{ failure() }}
        uses: actions/upload-artifact@v3
@@ -1,28 +1,42 @@
-name: Test the components of it
+name: Lint and validation

 "on":
-  push:
-    paths-ignore:
-      - .github/workflow/build.yml
-      - http/**
-      - README.box.md
-      - README.md
-      - LICENSE
  pull_request:
  workflow_dispatch:

 jobs:
  validate:
    name: Validate Packer templates are reasonable
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - uses: ./.github/actions/install_packer
-      - name: Run syntax check
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Packer
+        uses: ./.github/actions/install_packer
+
+      - name: Restore Packer plugin cache
+        uses: actions/cache/restore@v3
+        id: cache-restore
+        with:
+          path: ~/.config/packer/plugins
+          key: ${{ runner.os }}-${{ hashFiles('sources/build.pkr.hcl') }}
+
+      - name: Install Packer plugins
+        shell: bash
+        run: packer init sources
+
+      - name: Cache Packer plugins
+        uses: actions/cache/save@v3
+        if: always()
+        with:
+          path: ~/.config/packer/plugins
+          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
+
+      - name: Run validate check
        run: |
          set -exo pipefail
-          for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+          for vars in $(find distros/ -name '*.pkrvars.hcl*'); do
            if [[ "${vars}" == *.sh ]]; then
              "./${vars}"
              vars=${vars%.sh}
@@ -30,18 +44,17 @@ jobs:
            packer validate -except=upload -var-file=${vars} sources/
          done

+      - name: Run Packer format check
+        run: git ls-files | grep -E -e '.hcl$' | xargs -L 1 packer fmt -check
+
  lint:
-    name: Lint Ansible code
-    runs-on: ubuntu-22.04
+    name: Lint shell code
+    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
-      - name: Setup test environment
-        uses: ./.github/actions/setup
-        with:
-          ansible: true
-      - name: Run lint
+
+      - name: Install shellcheck
        run: |
-          yamllint -c ansible/yamllint.yml ansible .github distros
-          find distros/ -name user-data | xargs yamllint -c ansible/yamllint.yml
-          # TODO: Add ansible-lint run
+          shellcheck --version
+          shellcheck $(find . -name '*.sh')