Merge branch 'pre-cache-isos' into 'main'

Try pre-caching ISO files

See merge request greg/vms!9

.gitignore

@@ -18,6 +18,7 @@ __pycache__
 *.pyc
 *.pyo
 packer_cache
+cache
 .venv
 venv
 

.gitlab-ci-build.yml.in

@@ -11,8 +11,9 @@ stages:
           - sources/build.pkr.hcl
       paths:
         - ./packer_config/
-    - paths:
-        - packer_cache/*.iso
+    - key: isofiles
+      paths:
+        - cache/*.iso
   rules:
     - if: $CI_COMMIT_TAG
       variables:
@@ -31,7 +32,7 @@ stages:
     - ${tag}
   variables: &variables
     PACKER_CONFIG_DIR: ./packer_config/
-    PACKER_CACHE_DIR: ./packer_cache/
+    PACKER_CACHE_DIR: ./cache/
     PACKER_LOG: 1
     COMMAND: >
       packer build ${only}

.gitlab-ci.yml

@@ -1,8 +1,13 @@
 stages:
   - generate
   - lint
+  - cache
   - trigger
 
+variables:
+  PACKER_CONFIG_DIR: ./packer_config/
+  PACKER_CACHE_DIR: ./cache/
+
 generate:
   stage: generate
   artifacts:
@@ -13,26 +18,6 @@ generate:
       "./${vars}"
     done
 
-validate:
-  stage: lint
-  needs:
-    - generate
-  cache:
-    paths:
-      - /var/lib/gitlab-runner/.config/packer/plugins/
-  script: |-
-    set -exo pipefail
-    packer init sources
-    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
-      packer validate -except=upload -var-file=${vars} sources/
-    done
-
-shellcheck:
-  stage: lint
-  script: |-
-    shellcheck --version
-    shellcheck $(find . -name '*.sh')
-
 distros:
   stage: generate
   image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
@@ -45,6 +30,63 @@ distros:
     xonsh ./build.xsh -a --list
     echo "BUILD=$(date +'%Y.%m.%d.%H')" >> build.env
 
+validate:
+  stage: lint
+  needs:
+    - generate
+  cache:
+    - key:
+        files:
+          - sources/build.pkr.hcl
+      paths:
+        - ./packer_config/
+  script: |-
+    set -exo pipefail
+    packer init sources
+    for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+      packer validate -except=upload -var-file=${vars} sources/
+    done
+
+shellcheck:
+  stage: lint
+  script: |-
+    shellcheck --version
+    shellcheck $(find . -name '*.sh')
+
+# Downloads all the ISO files so that we have a single cache with all
+# of them present. This prevents a race condition where some of the
+# files were being downloaded multiple times due to parallel jobs, and
+# others were disappearing (e.g. daily builds for Rawhide) before we
+# could get to them
+downloads:
+  stage: cache
+  image: $CI_REGISTRY_DIRECT/greg/ci-images/fedora:latest
+  cache:
+    - key: isofiles
+      paths:
+        - cache/*.iso
+    - key:
+        files:
+          - sources/build.pkr.hcl
+      paths:
+        - ./packer_config/
+  script:
+    - set -exo pipefail
+    - packer init sources
+    - files=()
+    - |-
+      for vars in $(find distros/ -name '*.pkrvars.hcl'); do
+        file=$(echo var.iso.url | packer console -var-file=${vars} -config-type=hcl2 sources/)
+        base="$(basename "${file}")"
+        if [ ! -e "cache/${base}" ]; then
+          files+=("-O" "${file}")
+        fi
+      done
+    - mkdir -p cache
+    - pushd cache
+    - curl --parallel -L "${files[@]}"
+    - popd
+
 trigger:
   stage: trigger
   trigger:

build.xsh

@@ -84,6 +84,7 @@ if args.all or args.list or (args.distro and args.version and args.build):
         with open(".gitlab-ci-build.yml.in", "r") as fp:
             y = load(fp, Loader=Loader)
         for item in my_list:
+            print(item)
             if item.is_github:
                 y[ item.only[6:] ]["parallel"]["matrix"].append({"distro": item.var_path})
         with open(".gitlab-ci-build.yml", "w") as fp:

distros/opensuse/supports.yml

@@ -1,4 +1,4 @@
-15.6: &supp
+15.5: &supp
   - arch: amd64
     provider: qemu
   - arch: amd64
@@ -7,5 +7,5 @@
     provider: vmware-iso
   - arch: amd64
     provider: hyperv-iso
-15.5: *supp
+15.6:
 tumbleweed: *supp

sources/hyperv-iso.pkr.hcl

@@ -1,5 +1,6 @@
 source "hyperv-iso" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/hyperv/${local.name}"
 

sources/qemu.pkr.hcl

@@ -4,7 +4,8 @@ variable "qemu_accelerator" {
 }
 
 source "qemu" "amd64" {
-  iso_url          = var.iso.url
+  iso_urls         = [ "cache/${basename(var.iso.url)}", var.iso.url]
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/libvirt/${local.name}"
 

sources/virtualbox-iso.pkr.hcl

@@ -1,5 +1,6 @@
 source "virtualbox-iso" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/virtualbox/${local.name}"
 

sources/vmware-iso.pkr.hcl

@@ -1,5 +1,6 @@
 source "vmware-iso" "amd64" {
   iso_url          = var.iso.url
+  iso_target_path  = "cache/${basename(var.iso.url)}"
   iso_checksum     = var.iso.checksum
   output_directory = "output/vmware/${local.name}"