Convert the cache to golang

In the interest of performance, convert the fetcher and uploader to golang and use the minio library, rather than repeated calls to the server with the command line
2024-12-24 00:23:26 -06:00
parent 6c98a2aa39
commit f99c35b2d5
5 changed files with 276 additions and 24 deletions
@@ -2,9 +2,10 @@
  writeShellApplication,
  active,
  distroFile,
+  buildGoModule,
+  lib,

-  curl,
-  minio-client,
+  go,
  packer,

  ...
@@ -12,36 +13,36 @@

 let
  inherit (builtins) concatStringsSep map;
+  bin = buildGoModule {
+    pname = "cache";
+    version = "0.0.0";
+    src = ./cache;
+    vendorHash = "sha256-Mtq6BIlBNxI28eoLPfayov5uUqC4gMK+4ib6VKjyAlU=";
+    #vendorHash = lib.fakeHash;
+    meta.mainProgram = "main";
+  };
 in
 writeShellApplication {
  name = "cache-isos";
  runtimeInputs = [
-    curl
-    minio-client
+    go
    packer
  ];
  text =
    ''
-      push="''${1:-yes}"
-      if [ "$push" == "yes" ]; then
-      	mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET"
-      fi
-      mkdir -p cache
+      function list_isos {
    ''
    + (concatStringsSep "\n" (
      map (d: ''
-        url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
-        sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
-        distro=$(echo var.distro | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/)
-        # shellcheck disable=SC2143
-        if [ -z "$(mc ls "storage/isos/cache/$distro" | grep "$sha")" ]; then
-        	curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url"
-        	if [ "$push" == "yes" ]; then
-        		mc put "cache/$sha.iso" "storage/isos/cache/$distro/$sha.iso"
-        	fi
-        else
-        	echo "Skipping ${d.distro} - cache/$sha.iso"
-        fi
-      '') active
-    ));
+        url=$(echo var.iso.url | packer console -var-file="${d}" -config-type=hcl2 sources/)
+        sha=$(echo var.iso.checksum | packer console -var-file="${d}" -config-type=hcl2 sources/)
+        distro=$(echo var.distro | packer console -var-file="${d}" -config-type=hcl2 sources/)
+        echo "$url" "$sha" "$distro"
+      '') (lib.unique (lib.map distroFile active))
+    ))
+    + ''
+      }
+
+      list_isos | ${lib.getExe bin}
+    '';
 }
@@ -0,0 +1,190 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/schollz/progressbar/v3"
+)
+
+func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) {
+	var contents []string
+
+	// Create a done channel to control the listing
+	ctx := context.Background()
+
+	// List all objects from bucket
+	for object := range minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
+		Recursive: true,
+	}) {
+		if object.Err != nil {
+			return nil, object.Err
+		}
+		contents = append(contents, object.Key)
+	}
+
+	return contents, nil
+}
+
+type IsoInfo struct {
+	Url    string
+	Hash   string
+	Distro string
+}
+
+func processStdin() []IsoInfo {
+	var isos []IsoInfo
+	scanner := bufio.NewScanner(os.Stdin)
+
+	for scanner.Scan() {
+		parts := strings.Split(scanner.Text(), " ")
+		if len(parts) == 3 {
+			iso := IsoInfo{
+				Url:    parts[0],
+				Hash:   parts[1],
+				Distro: parts[2],
+			}
+			isos = append(isos, iso)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Println("Error reading from stdin:", err)
+	}
+
+	return isos
+}
+
+// CustomReader wraps an io.Reader with a progress bar
+type CustomReader struct {
+	Reader      io.Reader
+	ProgressBar *progressbar.ProgressBar
+}
+
+func (r *CustomReader) Read(p []byte) (int, error) {
+	n, err := r.Reader.Read(p)
+	if n > 0 {
+		r.ProgressBar.Add(n)
+	}
+	return n, err
+}
+
+func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error {
+	// Create the destination file path
+	destPath := "cache/" + iso.Hash + ".iso"
+
+	// Download the ISO file
+	resp, err := http.Get(iso.Url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	//
+	// Create download progress bar
+	downloadBar := progressbar.DefaultBytes(
+		resp.ContentLength,
+		"Downloading: "+iso.Url,
+	)
+
+	// Create the destination file
+	destFile, err := os.Create(destPath)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	// Create a wrapped reader with progress bar
+	reader := io.TeeReader(resp.Body, downloadBar)
+
+	// Copy the downloaded content to the file
+	_, err = io.Copy(destFile, reader)
+	if err != nil {
+		return err
+	}
+
+	// Reopen file for uploading
+	uploadFile, err := os.Open(destPath)
+	if err != nil {
+		return err
+	}
+	defer uploadFile.Close()
+
+	// Get file info for content-length
+	fileInfo, err := uploadFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	// Create upload progress bar
+	uploadBar := progressbar.DefaultBytes(
+		fileInfo.Size(),
+		"Uploading: "+destPath,
+	)
+
+	// Create wrapped reader for upload progress
+	uploadReader := &CustomReader{
+		Reader:      uploadFile,
+		ProgressBar: uploadBar,
+	}
+
+	// Upload the file to MinIO
+	_, err = minioClient.PutObject(context.Background(), "isos", "cache/"+iso.Hash+".iso", uploadReader, fileInfo.Size(), minio.PutObjectOptions{
+		ContentType: "application/x-iso9660-image",
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	endpoint := os.Getenv("STORAGE_URL")
+	accessKeyID := "root"
+	secretAccessKey := os.Getenv("MINIO_SECRET")
+	if secretAccessKey == "" {
+		log.Fatalln("MINIO_SECRET environment variable not set")
+	}
+	useSSL := false
+
+	// Initialize minio client
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	// List contents of the "isos" bucket
+	contents, err := listBucketContents(minioClient, "isos")
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	isos := processStdin()
+
+	for _, iso := range isos {
+		// Check if ISO exists in Minio
+		isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso"
+		exists := false
+
+		for _, item := range contents {
+			if item == isoPath {
+				exists = true
+				break
+			}
+		}
+
+		if !exists {
+			downloadAndUploadIso(minioClient, iso)
+		}
+	}
+}
@@ -0,0 +1,23 @@
+module main
+
+go 1.22
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.82 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	github.com/schollz/progressbar/v3 v3.17.1 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/term v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+)
@@ -0,0 +1,38 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
+github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U=
+github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
+golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=