From f99c35b2d5ed778d50dd0a699b315ae4229e96ce Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 00:23:26 -0600 Subject: [PATCH 1/6] Convert the cache to golang In the interest of performance, convert the fetcher and uploader to golang and use the minio library, rather than repeated calls to the server with the command line --- flake.nix | 2 +- nix/cache.nix | 47 +++++------ nix/cache/cache.go | 190 +++++++++++++++++++++++++++++++++++++++++++++ nix/cache/go.mod | 23 ++++++ nix/cache/go.sum | 38 +++++++++ 5 files changed, 276 insertions(+), 24 deletions(-) create mode 100644 nix/cache/cache.go create mode 100644 nix/cache/go.mod create mode 100644 nix/cache/go.sum diff --git a/flake.nix b/flake.nix index 6d6c572..d74d5c1 100644 --- a/flake.nix +++ b/flake.nix @@ -77,7 +77,7 @@ env = { EFI_DIR = "${pkgs.OVMF.fd}/FV/"; PACKER_CONFIG_DIR = "./packer_config"; - STORAGE_URL = "http://s3.thehellings.lan:9000"; + STORAGE_URL = "s3.thehellings.lan:9000"; }; }; diff --git a/nix/cache.nix b/nix/cache.nix index 6b0c4c5..0adf7e5 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -2,9 +2,10 @@ writeShellApplication, active, distroFile, + buildGoModule, + lib, - curl, - minio-client, + go, packer, ... @@ -12,36 +13,36 @@ let inherit (builtins) concatStringsSep map; + bin = buildGoModule { + pname = "cache"; + version = "0.0.0"; + src = ./cache; + vendorHash = "sha256-Mtq6BIlBNxI28eoLPfayov5uUqC4gMK+4ib6VKjyAlU="; + #vendorHash = lib.fakeHash; + meta.mainProgram = "main"; + }; in writeShellApplication { name = "cache-isos"; runtimeInputs = [ - curl - minio-client + go packer ]; text = '' - push="''${1:-yes}" - if [ "$push" == "yes" ]; then - mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" - fi - mkdir -p cache + function list_isos { '' + (concatStringsSep "\n" ( map (d: '' - url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - distro=$(echo var.distro | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - # shellcheck disable=SC2143 - if [ -z "$(mc ls "storage/isos/cache/$distro" | grep "$sha")" ]; then - curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" - if [ "$push" == "yes" ]; then - mc put "cache/$sha.iso" "storage/isos/cache/$distro/$sha.iso" - fi - else - echo "Skipping ${d.distro} - cache/$sha.iso" - fi - '') active - )); + url=$(echo var.iso.url | packer console -var-file="${d}" -config-type=hcl2 sources/) + sha=$(echo var.iso.checksum | packer console -var-file="${d}" -config-type=hcl2 sources/) + distro=$(echo var.distro | packer console -var-file="${d}" -config-type=hcl2 sources/) + echo "$url" "$sha" "$distro" + '') (lib.unique (lib.map distroFile active)) + )) + + '' + } + + list_isos | ${lib.getExe bin} + ''; } diff --git a/nix/cache/cache.go b/nix/cache/cache.go new file mode 100644 index 0000000..c1850aa --- /dev/null +++ b/nix/cache/cache.go @@ -0,0 +1,190 @@ +package main + +import ( + "bufio" + "context" + "io" + "log" + "net/http" + "os" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/schollz/progressbar/v3" +) + +func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) { + var contents []string + + // Create a done channel to control the listing + ctx := context.Background() + + // List all objects from bucket + for object := range minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Recursive: true, + }) { + if object.Err != nil { + return nil, object.Err + } + contents = append(contents, object.Key) + } + + return contents, nil +} + +type IsoInfo struct { + Url string + Hash string + Distro string +} + +func processStdin() []IsoInfo { + var isos []IsoInfo + scanner := bufio.NewScanner(os.Stdin) + + for scanner.Scan() { + parts := strings.Split(scanner.Text(), " ") + if len(parts) == 3 { + iso := IsoInfo{ + Url: parts[0], + Hash: parts[1], + Distro: parts[2], + } + isos = append(isos, iso) + } + } + + if err := scanner.Err(); err != nil { + log.Println("Error reading from stdin:", err) + } + + return isos +} + +// CustomReader wraps an io.Reader with a progress bar +type CustomReader struct { + Reader io.Reader + ProgressBar *progressbar.ProgressBar +} + +func (r *CustomReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + if n > 0 { + r.ProgressBar.Add(n) + } + return n, err +} + +func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { + // Create the destination file path + destPath := "cache/" + iso.Hash + ".iso" + + // Download the ISO file + resp, err := http.Get(iso.Url) + if err != nil { + return err + } + defer resp.Body.Close() + // + // Create download progress bar + downloadBar := progressbar.DefaultBytes( + resp.ContentLength, + "Downloading: "+iso.Url, + ) + + // Create the destination file + destFile, err := os.Create(destPath) + if err != nil { + return err + } + defer destFile.Close() + + // Create a wrapped reader with progress bar + reader := io.TeeReader(resp.Body, downloadBar) + + // Copy the downloaded content to the file + _, err = io.Copy(destFile, reader) + if err != nil { + return err + } + + // Reopen file for uploading + uploadFile, err := os.Open(destPath) + if err != nil { + return err + } + defer uploadFile.Close() + + // Get file info for content-length + fileInfo, err := uploadFile.Stat() + if err != nil { + return err + } + + // Create upload progress bar + uploadBar := progressbar.DefaultBytes( + fileInfo.Size(), + "Uploading: "+destPath, + ) + + // Create wrapped reader for upload progress + uploadReader := &CustomReader{ + Reader: uploadFile, + ProgressBar: uploadBar, + } + + // Upload the file to MinIO + _, err = minioClient.PutObject(context.Background(), "isos", "cache/"+iso.Hash+".iso", uploadReader, fileInfo.Size(), minio.PutObjectOptions{ + ContentType: "application/x-iso9660-image", + }) + if err != nil { + return err + } + + return nil +} + +func main() { + endpoint := os.Getenv("STORAGE_URL") + accessKeyID := "root" + secretAccessKey := os.Getenv("MINIO_SECRET") + if secretAccessKey == "" { + log.Fatalln("MINIO_SECRET environment variable not set") + } + useSSL := false + + // Initialize minio client + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + log.Fatalln(err) + } + + // List contents of the "isos" bucket + contents, err := listBucketContents(minioClient, "isos") + if err != nil { + log.Fatalln(err) + } + + isos := processStdin() + + for _, iso := range isos { + // Check if ISO exists in Minio + isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" + exists := false + + for _, item := range contents { + if item == isoPath { + exists = true + break + } + } + + if !exists { + downloadAndUploadIso(minioClient, iso) + } + } +} diff --git a/nix/cache/go.mod b/nix/cache/go.mod new file mode 100644 index 0000000..864b321 --- /dev/null +++ b/nix/cache/go.mod @@ -0,0 +1,23 @@ +module main + +go 1.22 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.82 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/schollz/progressbar/v3 v3.17.1 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect +) diff --git a/nix/cache/go.sum b/nix/cache/go.sum new file mode 100644 index 0000000..cc83aef --- /dev/null +++ b/nix/cache/go.sum @@ -0,0 +1,38 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo= +github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= +github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= From cc78eea201c270c513007cd44eabe79e4c68ede2 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 00:49:09 -0600 Subject: [PATCH 2/6] Only build Hyper V on releases --- nix/continue.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/continue.nix b/nix/continue.nix index f21627e..e4f3223 100644 --- a/nix/continue.nix +++ b/nix/continue.nix @@ -70,6 +70,7 @@ let variables = general.variables // { COMMAND = "packer build -only=\"hyperv-iso.amd64\" -var-file=\"\${distro}\" -var build=\${BUILD} -var cpus=2 -var memory=4096 \${EXCEPT} sources"; }; + rules = [ { "if" = "$CI_COMMIT_TAG"; } ]; script = [ "packer init sources" "echo $env:COMMAND > build.ps1" From 8b2e4b252ea82ce3fd43294e15f1c3aaea9e11a8 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 00:49:56 -0600 Subject: [PATCH 3/6] Fedora 39 EOL Fedora 39 end of life on November 26, 2024. Fixes #21 --- distros/fedora/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distros/fedora/default.nix b/distros/fedora/default.nix index 59e858b..153304f 100644 --- a/distros/fedora/default.nix +++ b/distros/fedora/default.nix @@ -12,7 +12,7 @@ { version = "39"; arch = "amd64"; - eol = false; + eol = true; } { version = "40"; From f75558bfb7aa23fc052f4757e683d8f969b49b59 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 01:00:23 -0600 Subject: [PATCH 4/6] Update S3 URL Since the STORAGE_URL variable no longer has the `http://` portion of the address, we have to add it into the hcl files --- sources/inputs.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/inputs.pkr.hcl b/sources/inputs.pkr.hcl index 230548e..2fcc09d 100644 --- a/sources/inputs.pkr.hcl +++ b/sources/inputs.pkr.hcl @@ -127,7 +127,7 @@ locals { version = var.version build = var.build }) - iso_location = "s3::${var.storage_url}/isos/cache/${var.distro}/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" + iso_location = "s3::http://${var.storage_url}/isos/cache/${var.distro}/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" cpus = var.cpus memory = var.memory From 0c5cdb35b34336d290edeb921f82253d6e77ea9f Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 01:09:10 -0600 Subject: [PATCH 5/6] Remove the progress bar and fix the downloader The progress bar *might* be causing problems during the upload. So we remove it and allow the progress to be mysterious to the user Return a non-zero error code if there is a failure Make the cache directory first... --- nix/cache.nix | 1 + nix/cache/cache.go | 52 ++++++++++++---------------------------------- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/nix/cache.nix b/nix/cache.nix index 0adf7e5..239551e 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -30,6 +30,7 @@ writeShellApplication { ]; text = '' + set -o pipefail function list_isos { '' + (concatStringsSep "\n" ( diff --git a/nix/cache/cache.go b/nix/cache/cache.go index c1850aa..f7e6fae 100644 --- a/nix/cache/cache.go +++ b/nix/cache/cache.go @@ -11,7 +11,6 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/schollz/progressbar/v3" ) func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) { @@ -62,36 +61,22 @@ func processStdin() []IsoInfo { return isos } -// CustomReader wraps an io.Reader with a progress bar -type CustomReader struct { - Reader io.Reader - ProgressBar *progressbar.ProgressBar -} - -func (r *CustomReader) Read(p []byte) (int, error) { - n, err := r.Reader.Read(p) - if n > 0 { - r.ProgressBar.Add(n) - } - return n, err -} - func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { // Create the destination file path destPath := "cache/" + iso.Hash + ".iso" + // Create cache directory if it doesn't exist + err := os.MkdirAll("cache", 0755) + if err != nil { + return err + } + log.Println("Downloading " + iso.Url) // Download the ISO file resp, err := http.Get(iso.Url) if err != nil { return err } defer resp.Body.Close() - // - // Create download progress bar - downloadBar := progressbar.DefaultBytes( - resp.ContentLength, - "Downloading: "+iso.Url, - ) // Create the destination file destFile, err := os.Create(destPath) @@ -100,11 +85,8 @@ func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { } defer destFile.Close() - // Create a wrapped reader with progress bar - reader := io.TeeReader(resp.Body, downloadBar) - // Copy the downloaded content to the file - _, err = io.Copy(destFile, reader) + _, err = io.Copy(destFile, resp.Body) if err != nil { return err } @@ -122,20 +104,9 @@ func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { return err } - // Create upload progress bar - uploadBar := progressbar.DefaultBytes( - fileInfo.Size(), - "Uploading: "+destPath, - ) - - // Create wrapped reader for upload progress - uploadReader := &CustomReader{ - Reader: uploadFile, - ProgressBar: uploadBar, - } - + log.Println("Uploading " + destPath) // Upload the file to MinIO - _, err = minioClient.PutObject(context.Background(), "isos", "cache/"+iso.Hash+".iso", uploadReader, fileInfo.Size(), minio.PutObjectOptions{ + _, err = minioClient.PutObject(context.Background(), "isos", "cache/"+iso.Hash+".iso", uploadFile, fileInfo.Size(), minio.PutObjectOptions{ ContentType: "application/x-iso9660-image", }) if err != nil { @@ -184,7 +155,10 @@ func main() { } if !exists { - downloadAndUploadIso(minioClient, iso) + if err := downloadAndUploadIso(minioClient, iso); err != nil { + log.Println("Error downloading/uploading ISO:", err) + os.Exit(1) + } } } } From 82718cbc4c17681931ad5b1cf66d357f28079a04 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 24 Dec 2024 10:30:06 -0600 Subject: [PATCH 6/6] Add some debugging output --- nix/cache/cache.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/cache/cache.go b/nix/cache/cache.go index f7e6fae..bd3f03d 100644 --- a/nix/cache/cache.go +++ b/nix/cache/cache.go @@ -105,8 +105,9 @@ func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { } log.Println("Uploading " + destPath) + uploadPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" // Upload the file to MinIO - _, err = minioClient.PutObject(context.Background(), "isos", "cache/"+iso.Hash+".iso", uploadFile, fileInfo.Size(), minio.PutObjectOptions{ + _, err = minioClient.PutObject(context.Background(), "isos", uploadPath, uploadFile, fileInfo.Size(), minio.PutObjectOptions{ ContentType: "application/x-iso9660-image", }) if err != nil { @@ -118,6 +119,7 @@ func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { func main() { endpoint := os.Getenv("STORAGE_URL") + log.Println("Using endpoint:", endpoint) accessKeyID := "root" secretAccessKey := os.Getenv("MINIO_SECRET") if secretAccessKey == "" {