From ea5118889760f5c509336046de0000d2ebe0adab Mon Sep 17 00:00:00 2001
From: Greg Hellings <greg.hellings@gmail.com>
Date: Wed, 10 Apr 2024 12:56:52 -0500
Subject: [PATCH] Try pre-caching ISO files

Since the builds themselves get run in a big parallel morass of
nonsense, this causes cache clobbering all over. So now I move the
downloading to the pre-trigger stage and hopefully we can use
the pre-download step to cache things properly. Hopefully this will also
prevent issues where dynamically generated download files are missing
for later runs.
---
 .gitignore                     |  1 +
 .gitlab-ci-build.yml.in        |  7 ++--
 .gitlab-ci.yml                 | 76 +++++++++++++++++++++++++---------
 sources/hyperv-iso.pkr.hcl     |  1 +
 sources/qemu.pkr.hcl           |  1 +
 sources/virtualbox-iso.pkr.hcl |  1 +
 sources/vmware-iso.pkr.hcl     |  1 +
 7 files changed, 65 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 104b1e3..5012c47 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@ __pycache__
 *.pyc
 *.pyo
 packer_cache
+cache
 .venv
 venv
 
diff --git a/.gitlab-ci-build.yml.in b/.gitlab-ci-build.yml.in
index 3495756..6da64c2 100644
--- a/.gitlab-ci-build.yml.in
+++ b/.gitlab-ci-build.yml.in
@@ -11,8 +11,9 @@ stages:
           - sources/build.pkr.hcl
       paths:
         - ./packer_config/
-    - paths:
-        - packer_cache/*.iso
+    - key: isofiles
+      paths:
+        - cache/*.iso
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -31,7 +32,7 @@ stages:
     - ${tag}
   variables: &variables
     PACKER_CONFIG_DIR: ./packer_config/
-    PACKER_CACHE_DIR: ./packer_cache/
+    PACKER_CACHE_DIR: ./cache/
     PACKER_LOG: 1
     COMMAND: >
       packer build ${only}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4ae619d..d8baee3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,13 @@
 stages:
   - generate
   - lint
+  - cache
   - trigger
 
+variables:
+  PACKER_CONFIG_DIR: ./packer_config/
+  PACKER_CACHE_DIR: ./cache/
+
 generate:
   stage: generate
   artifacts:
@@ -13,26 +18,6 @@ generate:
       "./${vars}"
     done
 
-validate:
-  stage: lint
-  needs:
-    - generate
-  cache:
-    paths:
-      - /var/lib/gitlab-runner/.config/packer/plugins/
-  script: |-
-    set -exo pipefail
-    packer init sources
-    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
-      packer validate -except=upload -var-file=${vars} sources/
-    done
-
-shellcheck:
-  stage: lint
-  script: |-
-    shellcheck --version
-    shellcheck $(find . -name '*.sh')
-
 distros:
   stage: generate
   image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
@@ -45,6 +30,57 @@ distros:
     xonsh ./build.xsh -a --list
     echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env
 
+validate:
+  stage: lint
+  needs:
+    - generate
+  cache:
+    - key:
+        files:
+          - sources/build.pkr.hcl
+      paths:
+        - ./packer_config/
+  script: |-
+    set -exo pipefail
+    packer init sources
+    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+      packer validate -except=upload -var-file=${vars} sources/
+    done
+
+shellcheck:
+  stage: lint
+  script: |-
+    shellcheck --version
+    shellcheck $(find . -name '*.sh')
+
+# Downloads all the ISO files so that we have a single cache with all
+# of them present. This prevents a race condition where some of the
+# files were being downloaded multiple times due to parallel jobs, and
+# others were disappearing (e.g. daily builds for Rawhide) before we
+# could get to them
+downloads:
+  stage: cache
+  image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
+  cache:
+    - key: isofiles
+      paths:
+        - cache/*.iso
+    - key:
+        files:
+          - sources/build.pkr.hcl
+        paths:
+          - ./packer_config/
+  script:
+    - set -exo pipefail
+    - packer init sources
+    - |-
+      for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+        file=$(echo var.iso.url | packer console -var-file=${vars} -config-type=hcl2 sources/)
+        pushd cache
+        curl -O "${file}"
+        popd
+      done
+
 trigger:
   stage: trigger
   trigger:
diff --git a/sources/hyperv-iso.pkr.hcl b/sources/hyperv-iso.pkr.hcl
index 1d1691d..cd06012 100644
--- a/sources/hyperv-iso.pkr.hcl
+++ b/sources/hyperv-iso.pkr.hcl
@@ -1,5 +1,6 @@
 source "hyperv-iso" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/hyperv/${local.name}"
 
diff --git a/sources/qemu.pkr.hcl b/sources/qemu.pkr.hcl
index ec70ccc..9a69ff3 100644
--- a/sources/qemu.pkr.hcl
+++ b/sources/qemu.pkr.hcl
@@ -5,6 +5,7 @@ variable "qemu_accelerator" {
 
 source "qemu" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/libvirt/${local.name}"
 
diff --git a/sources/virtualbox-iso.pkr.hcl b/sources/virtualbox-iso.pkr.hcl
index efee8c1..5848ff8 100644
--- a/sources/virtualbox-iso.pkr.hcl
+++ b/sources/virtualbox-iso.pkr.hcl
@@ -1,5 +1,6 @@
 source "virtualbox-iso" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/virtualbox/${local.name}"
 
diff --git a/sources/vmware-iso.pkr.hcl b/sources/vmware-iso.pkr.hcl
index 065eae2..687b7ac 100644
--- a/sources/vmware-iso.pkr.hcl
+++ b/sources/vmware-iso.pkr.hcl
@@ -1,5 +1,6 @@
 source "vmware-iso" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/vmware/${local.name}"