From f99c35b2d5ed778d50dd0a699b315ae4229e96ce Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 00:23:26 -0600 Subject: [PATCH] Convert the cache to golang In the interest of performance, convert the fetcher and uploader to golang and use the minio library, rather than repeated calls to the server with the command line --- flake.nix | 2 +- nix/cache.nix | 47 +++++------ nix/cache/cache.go | 190 +++++++++++++++++++++++++++++++++++++++++++++ nix/cache/go.mod | 23 ++++++ nix/cache/go.sum | 38 +++++++++ 5 files changed, 276 insertions(+), 24 deletions(-) create mode 100644 nix/cache/cache.go create mode 100644 nix/cache/go.mod create mode 100644 nix/cache/go.sum diff --git a/flake.nix b/flake.nix index 6d6c572..d74d5c1 100644 --- a/flake.nix +++ b/flake.nix @@ -77,7 +77,7 @@ env = { EFI_DIR = "${pkgs.OVMF.fd}/FV/"; PACKER_CONFIG_DIR = "./packer_config"; - STORAGE_URL = "http://s3.thehellings.lan:9000"; + STORAGE_URL = "s3.thehellings.lan:9000"; }; }; diff --git a/nix/cache.nix b/nix/cache.nix index 6b0c4c5..0adf7e5 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -2,9 +2,10 @@ writeShellApplication, active, distroFile, + buildGoModule, + lib, - curl, - minio-client, + go, packer, ... @@ -12,36 +13,36 @@ let inherit (builtins) concatStringsSep map; + bin = buildGoModule { + pname = "cache"; + version = "0.0.0"; + src = ./cache; + vendorHash = "sha256-Mtq6BIlBNxI28eoLPfayov5uUqC4gMK+4ib6VKjyAlU="; + #vendorHash = lib.fakeHash; + meta.mainProgram = "main"; + }; in writeShellApplication { name = "cache-isos"; runtimeInputs = [ - curl - minio-client + go packer ]; text = '' - push="''${1:-yes}" - if [ "$push" == "yes" ]; then - mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" - fi - mkdir -p cache + function list_isos { '' + (concatStringsSep "\n" ( map (d: '' - url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - distro=$(echo var.distro | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - # shellcheck disable=SC2143 - if [ -z "$(mc ls "storage/isos/cache/$distro" | grep "$sha")" ]; then - curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" - if [ "$push" == "yes" ]; then - mc put "cache/$sha.iso" "storage/isos/cache/$distro/$sha.iso" - fi - else - echo "Skipping ${d.distro} - cache/$sha.iso" - fi - '') active - )); + url=$(echo var.iso.url | packer console -var-file="${d}" -config-type=hcl2 sources/) + sha=$(echo var.iso.checksum | packer console -var-file="${d}" -config-type=hcl2 sources/) + distro=$(echo var.distro | packer console -var-file="${d}" -config-type=hcl2 sources/) + echo "$url" "$sha" "$distro" + '') (lib.unique (lib.map distroFile active)) + )) + + '' + } + + list_isos | ${lib.getExe bin} + ''; } diff --git a/nix/cache/cache.go b/nix/cache/cache.go new file mode 100644 index 0000000..c1850aa --- /dev/null +++ b/nix/cache/cache.go @@ -0,0 +1,190 @@ +package main + +import ( + "bufio" + "context" + "io" + "log" + "net/http" + "os" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/schollz/progressbar/v3" +) + +func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) { + var contents []string + + // Create a done channel to control the listing + ctx := context.Background() + + // List all objects from bucket + for object := range minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Recursive: true, + }) { + if object.Err != nil { + return nil, object.Err + } + contents = append(contents, object.Key) + } + + return contents, nil +} + +type IsoInfo struct { + Url string + Hash string + Distro string +} + +func processStdin() []IsoInfo { + var isos []IsoInfo + scanner := bufio.NewScanner(os.Stdin) + + for scanner.Scan() { + parts := strings.Split(scanner.Text(), " ") + if len(parts) == 3 { + iso := IsoInfo{ + Url: parts[0], + Hash: parts[1], + Distro: parts[2], + } + isos = append(isos, iso) + } + } + + if err := scanner.Err(); err != nil { + log.Println("Error reading from stdin:", err) + } + + return isos +} + +// CustomReader wraps an io.Reader with a progress bar +type CustomReader struct { + Reader io.Reader + ProgressBar *progressbar.ProgressBar +} + +func (r *CustomReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + if n > 0 { + r.ProgressBar.Add(n) + } + return n, err +} + +func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { + // Create the destination file path + destPath := "cache/" + iso.Hash + ".iso" + + // Download the ISO file + resp, err := http.Get(iso.Url) + if err != nil { + return err + } + defer resp.Body.Close() + // + // Create download progress bar + downloadBar := progressbar.DefaultBytes( + resp.ContentLength, + "Downloading: "+iso.Url, + ) + + // Create the destination file + destFile, err := os.Create(destPath) + if err != nil { + return err + } + defer destFile.Close() + + // Create a wrapped reader with progress bar + reader := io.TeeReader(resp.Body, downloadBar) + + // Copy the downloaded content to the file + _, err = io.Copy(destFile, reader) + if err != nil { + return err + } + + // Reopen file for uploading + uploadFile, err := os.Open(destPath) + if err != nil { + return err + } + defer uploadFile.Close() + + // Get file info for content-length + fileInfo, err := uploadFile.Stat() + if err != nil { + return err + } + + // Create upload progress bar + uploadBar := progressbar.DefaultBytes( + fileInfo.Size(), + "Uploading: "+destPath, + ) + + // Create wrapped reader for upload progress + uploadReader := &CustomReader{ + Reader: uploadFile, + ProgressBar: uploadBar, + } + + // Upload the file to MinIO + _, err = minioClient.PutObject(context.Background(), "isos", "cache/"+iso.Hash+".iso", uploadReader, fileInfo.Size(), minio.PutObjectOptions{ + ContentType: "application/x-iso9660-image", + }) + if err != nil { + return err + } + + return nil +} + +func main() { + endpoint := os.Getenv("STORAGE_URL") + accessKeyID := "root" + secretAccessKey := os.Getenv("MINIO_SECRET") + if secretAccessKey == "" { + log.Fatalln("MINIO_SECRET environment variable not set") + } + useSSL := false + + // Initialize minio client + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + log.Fatalln(err) + } + + // List contents of the "isos" bucket + contents, err := listBucketContents(minioClient, "isos") + if err != nil { + log.Fatalln(err) + } + + isos := processStdin() + + for _, iso := range isos { + // Check if ISO exists in Minio + isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" + exists := false + + for _, item := range contents { + if item == isoPath { + exists = true + break + } + } + + if !exists { + downloadAndUploadIso(minioClient, iso) + } + } +} diff --git a/nix/cache/go.mod b/nix/cache/go.mod new file mode 100644 index 0000000..864b321 --- /dev/null +++ b/nix/cache/go.mod @@ -0,0 +1,23 @@ +module main + +go 1.22 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.82 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/schollz/progressbar/v3 v3.17.1 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect +) diff --git a/nix/cache/go.sum b/nix/cache/go.sum new file mode 100644 index 0000000..cc83aef --- /dev/null +++ b/nix/cache/go.sum @@ -0,0 +1,38 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo= +github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= +github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=