Merge branch 'cache-in-go' into 'main'
Convert the cache to golang Closes #21 See merge request greg/vms!15
This commit is contained in:
Greg Hellings
@@ -12,7 +12,7 @@
|
||||
{
|
||||
version = "39";
|
||||
arch = "amd64";
|
||||
eol = false;
|
||||
eol = true;
|
||||
}
|
||||
{
|
||||
version = "40";
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
env = {
|
||||
EFI_DIR = "${pkgs.OVMF.fd}/FV/";
|
||||
PACKER_CONFIG_DIR = "./packer_config";
|
||||
STORAGE_URL = "http://s3.thehellings.lan:9000";
|
||||
STORAGE_URL = "s3.thehellings.lan:9000";
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
+25
-23
@@ -2,9 +2,10 @@
|
||||
writeShellApplication,
|
||||
active,
|
||||
distroFile,
|
||||
buildGoModule,
|
||||
lib,
|
||||
|
||||
curl,
|
||||
minio-client,
|
||||
go,
|
||||
packer,
|
||||
|
||||
...
|
||||
@@ -12,36 +13,37 @@
|
||||
|
||||
let
|
||||
inherit (builtins) concatStringsSep map;
|
||||
bin = buildGoModule {
|
||||
pname = "cache";
|
||||
version = "0.0.0";
|
||||
src = ./cache;
|
||||
vendorHash = "sha256-Mtq6BIlBNxI28eoLPfayov5uUqC4gMK+4ib6VKjyAlU=";
|
||||
#vendorHash = lib.fakeHash;
|
||||
meta.mainProgram = "main";
|
||||
};
|
||||
in
|
||||
writeShellApplication {
|
||||
name = "cache-isos";
|
||||
runtimeInputs = [
|
||||
curl
|
||||
minio-client
|
||||
go
|
||||
packer
|
||||
];
|
||||
text =
|
||||
''
|
||||
push="''${1:-yes}"
|
||||
if [ "$push" == "yes" ]; then
|
||||
mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET"
|
||||
fi
|
||||
mkdir -p cache
|
||||
set -o pipefail
|
||||
function list_isos {
|
||||
''
|
||||
+ (concatStringsSep "\n" (
|
||||
map (d: ''
|
||||
url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
|
||||
sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
|
||||
distro=$(echo var.distro | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
|
||||
# shellcheck disable=SC2143
|
||||
if [ -z "$(mc ls "storage/isos/cache/$distro" | grep "$sha")" ]; then
|
||||
curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url"
|
||||
if [ "$push" == "yes" ]; then
|
||||
mc put "cache/$sha.iso" "storage/isos/cache/$distro/$sha.iso"
|
||||
fi
|
||||
else
|
||||
echo "Skipping ${d.distro} - cache/$sha.iso"
|
||||
fi
|
||||
'') active
|
||||
));
|
||||
url=$(echo var.iso.url | packer console -var-file="${d}" -config-type=hcl2 sources/)
|
||||
sha=$(echo var.iso.checksum | packer console -var-file="${d}" -config-type=hcl2 sources/)
|
||||
distro=$(echo var.distro | packer console -var-file="${d}" -config-type=hcl2 sources/)
|
||||
echo "$url" "$sha" "$distro"
|
||||
'') (lib.unique (lib.map distroFile active))
|
||||
))
|
||||
+ ''
|
||||
}
|
||||
|
||||
list_isos | ${lib.getExe bin}
|
||||
'';
|
||||
}
|
||||
|
||||
Vendored
+166
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) {
|
||||
var contents []string
|
||||
|
||||
// Create a done channel to control the listing
|
||||
ctx := context.Background()
|
||||
|
||||
// List all objects from bucket
|
||||
for object := range minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
|
||||
Recursive: true,
|
||||
}) {
|
||||
if object.Err != nil {
|
||||
return nil, object.Err
|
||||
}
|
||||
contents = append(contents, object.Key)
|
||||
}
|
||||
|
||||
return contents, nil
|
||||
}
|
||||
|
||||
type IsoInfo struct {
|
||||
Url string
|
||||
Hash string
|
||||
Distro string
|
||||
}
|
||||
|
||||
func processStdin() []IsoInfo {
|
||||
var isos []IsoInfo
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
|
||||
for scanner.Scan() {
|
||||
parts := strings.Split(scanner.Text(), " ")
|
||||
if len(parts) == 3 {
|
||||
iso := IsoInfo{
|
||||
Url: parts[0],
|
||||
Hash: parts[1],
|
||||
Distro: parts[2],
|
||||
}
|
||||
isos = append(isos, iso)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Println("Error reading from stdin:", err)
|
||||
}
|
||||
|
||||
return isos
|
||||
}
|
||||
|
||||
func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error {
|
||||
// Create the destination file path
|
||||
destPath := "cache/" + iso.Hash + ".iso"
|
||||
// Create cache directory if it doesn't exist
|
||||
err := os.MkdirAll("cache", 0755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Println("Downloading " + iso.Url)
|
||||
// Download the ISO file
|
||||
resp, err := http.Get(iso.Url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Create the destination file
|
||||
destFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
// Copy the downloaded content to the file
|
||||
_, err = io.Copy(destFile, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Reopen file for uploading
|
||||
uploadFile, err := os.Open(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer uploadFile.Close()
|
||||
|
||||
// Get file info for content-length
|
||||
fileInfo, err := uploadFile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Println("Uploading " + destPath)
|
||||
uploadPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso"
|
||||
// Upload the file to MinIO
|
||||
_, err = minioClient.PutObject(context.Background(), "isos", uploadPath, uploadFile, fileInfo.Size(), minio.PutObjectOptions{
|
||||
ContentType: "application/x-iso9660-image",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := os.Getenv("STORAGE_URL")
|
||||
log.Println("Using endpoint:", endpoint)
|
||||
accessKeyID := "root"
|
||||
secretAccessKey := os.Getenv("MINIO_SECRET")
|
||||
if secretAccessKey == "" {
|
||||
log.Fatalln("MINIO_SECRET environment variable not set")
|
||||
}
|
||||
useSSL := false
|
||||
|
||||
// Initialize minio client
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// List contents of the "isos" bucket
|
||||
contents, err := listBucketContents(minioClient, "isos")
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
isos := processStdin()
|
||||
|
||||
for _, iso := range isos {
|
||||
// Check if ISO exists in Minio
|
||||
isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso"
|
||||
exists := false
|
||||
|
||||
for _, item := range contents {
|
||||
if item == isoPath {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !exists {
|
||||
if err := downloadAndUploadIso(minioClient, iso); err != nil {
|
||||
log.Println("Error downloading/uploading ISO:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+23
@@ -0,0 +1,23 @@
|
||||
module main
|
||||
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/minio/minio-go/v7 v7.0.82 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/schollz/progressbar/v3 v3.17.1 // indirect
|
||||
golang.org/x/crypto v0.28.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.27.0 // indirect
|
||||
golang.org/x/term v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
)
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
|
||||
github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U=
|
||||
github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4=
|
||||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
|
||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
|
||||
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
|
||||
golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
@@ -70,6 +70,7 @@ let
|
||||
variables = general.variables // {
|
||||
COMMAND = "packer build -only=\"hyperv-iso.amd64\" -var-file=\"\${distro}\" -var build=\${BUILD} -var cpus=2 -var memory=4096 \${EXCEPT} sources";
|
||||
};
|
||||
rules = [ { "if" = "$CI_COMMIT_TAG"; } ];
|
||||
script = [
|
||||
"packer init sources"
|
||||
"echo $env:COMMAND > build.ps1"
|
||||
|
||||
@@ -127,7 +127,7 @@ locals {
|
||||
version = var.version
|
||||
build = var.build
|
||||
})
|
||||
iso_location = "s3::${var.storage_url}/isos/cache/${var.distro}/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}"
|
||||
iso_location = "s3::http://${var.storage_url}/isos/cache/${var.distro}/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}"
|
||||
|
||||
cpus = var.cpus
|
||||
memory = var.memory
|
||||
|
||||
Reference in New Issue
Block a user