Convert the cache to golang

In the interest of performance, convert the fetcher and uploader to
golang and use the minio library, rather than repeated calls to the
server with the command line
This commit is contained in:
Greg Hellings
2024-12-24 00:23:26 -06:00
parent 6c98a2aa39
commit f99c35b2d5
5 changed files with 276 additions and 24 deletions
+24 -23
View File
@@ -2,9 +2,10 @@
writeShellApplication,
active,
distroFile,
buildGoModule,
lib,
curl,
minio-client,
go,
packer,
...
@@ -12,36 +13,36 @@
let
inherit (builtins) concatStringsSep map;
bin = buildGoModule {
pname = "cache";
version = "0.0.0";
src = ./cache;
vendorHash = "sha256-Mtq6BIlBNxI28eoLPfayov5uUqC4gMK+4ib6VKjyAlU=";
#vendorHash = lib.fakeHash;
meta.mainProgram = "main";
};
in
writeShellApplication {
name = "cache-isos";
runtimeInputs = [
curl
minio-client
go
packer
];
text =
''
push="''${1:-yes}"
if [ "$push" == "yes" ]; then
mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET"
fi
mkdir -p cache
function list_isos {
''
+ (concatStringsSep "\n" (
map (d: ''
url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
distro=$(echo var.distro | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
# shellcheck disable=SC2143
if [ -z "$(mc ls "storage/isos/cache/$distro" | grep "$sha")" ]; then
curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url"
if [ "$push" == "yes" ]; then
mc put "cache/$sha.iso" "storage/isos/cache/$distro/$sha.iso"
fi
else
echo "Skipping ${d.distro} - cache/$sha.iso"
fi
'') active
));
url=$(echo var.iso.url | packer console -var-file="${d}" -config-type=hcl2 sources/)
sha=$(echo var.iso.checksum | packer console -var-file="${d}" -config-type=hcl2 sources/)
distro=$(echo var.distro | packer console -var-file="${d}" -config-type=hcl2 sources/)
echo "$url" "$sha" "$distro"
'') (lib.unique (lib.map distroFile active))
))
+ ''
}
list_isos | ${lib.getExe bin}
'';
}