Try pre-caching ISO files

Since the builds themselves get run in a big parallel morass of
nonsense, this causes cache clobbering all over. So now I move the
downloading to the pre-trigger stage and hopefully we can use
the pre-download step to cache things properly. Hopefully this will also
prevent issues where dynamically generated download files are missing
for later runs.

.gitlab-ci.yml

@@ -1,8 +1,13 @@
 stages:
   - generate
   - lint
+  - cache
   - trigger
 
+variables:
+  PACKER_CONFIG_DIR: ./packer_config/
+  PACKER_CACHE_DIR: ./cache/
+
 generate:
   stage: generate
   artifacts:
@@ -13,26 +18,6 @@ generate:
       "./${vars}"
     done
 
-validate:
-  stage: lint
-  needs:
-    - generate
-  cache:
-    paths:
-      - /var/lib/gitlab-runner/.config/packer/plugins/
-  script: |-
-    set -exo pipefail
-    packer init sources
-    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
-      packer validate -except=upload -var-file=${vars} sources/
-    done
-
-shellcheck:
-  stage: lint
-  script: |-
-    shellcheck --version
-    shellcheck $(find . -name '*.sh')
-
 distros:
   stage: generate
   image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
@@ -45,6 +30,57 @@ distros:
     xonsh ./build.xsh -a --list
     echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env
 
+validate:
+  stage: lint
+  needs:
+    - generate
+  cache:
+    - key:
+        files:
+          - sources/build.pkr.hcl
+      paths:
+        - ./packer_config/
+  script: |-
+    set -exo pipefail
+    packer init sources
+    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+      packer validate -except=upload -var-file=${vars} sources/
+    done
+
+shellcheck:
+  stage: lint
+  script: |-
+    shellcheck --version
+    shellcheck $(find . -name '*.sh')
+
+# Downloads all the ISO files so that we have a single cache with all
+# of them present. This prevents a race condition where some of the
+# files were being downloaded multiple times due to parallel jobs, and
+# others were disappearing (e.g. daily builds for Rawhide) before we
+# could get to them
+downloads:
+  stage: cache
+  image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
+  cache:
+    - key: isofiles
+      paths:
+        - cache/*.iso
+    - key:
+        files:
+          - sources/build.pkr.hcl
+        paths:
+          - ./packer_config/
+  script:
+    - set -exo pipefail
+    - packer init sources
+    - |-
+      for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+        file=$(echo var.iso.url | packer console -var-file=${vars} -config-type=hcl2 sources/)
+        pushd cache
+        curl -O "${file}"
+        popd
+      done
+
 trigger:
   stage: trigger
   trigger: