diff --git a/distros/fedora/default.nix b/distros/fedora/default.nix index 59e858b..153304f 100644 --- a/distros/fedora/default.nix +++ b/distros/fedora/default.nix @@ -12,7 +12,7 @@ { version = "39"; arch = "amd64"; - eol = false; + eol = true; } { version = "40"; diff --git a/flake.nix b/flake.nix index 6d6c572..d74d5c1 100644 --- a/flake.nix +++ b/flake.nix @@ -77,7 +77,7 @@ env = { EFI_DIR = "${pkgs.OVMF.fd}/FV/"; PACKER_CONFIG_DIR = "./packer_config"; - STORAGE_URL = "http://s3.thehellings.lan:9000"; + STORAGE_URL = "s3.thehellings.lan:9000"; }; }; diff --git a/nix/cache.nix b/nix/cache.nix index 6b0c4c5..239551e 100644 --- a/nix/cache.nix +++ b/nix/cache.nix @@ -2,9 +2,10 @@ writeShellApplication, active, distroFile, + buildGoModule, + lib, - curl, - minio-client, + go, packer, ... @@ -12,36 +13,37 @@ let inherit (builtins) concatStringsSep map; + bin = buildGoModule { + pname = "cache"; + version = "0.0.0"; + src = ./cache; + vendorHash = "sha256-Mtq6BIlBNxI28eoLPfayov5uUqC4gMK+4ib6VKjyAlU="; + #vendorHash = lib.fakeHash; + meta.mainProgram = "main"; + }; in writeShellApplication { name = "cache-isos"; runtimeInputs = [ - curl - minio-client + go packer ]; text = '' - push="''${1:-yes}" - if [ "$push" == "yes" ]; then - mc alias set storage "$STORAGE_URL" root "$MINIO_SECRET" - fi - mkdir -p cache + set -o pipefail + function list_isos { '' + (concatStringsSep "\n" ( map (d: '' - url=$(echo var.iso.url | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - sha=$(echo var.iso.checksum | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - distro=$(echo var.distro | packer console -var-file="${distroFile d}" -config-type=hcl2 sources/) - # shellcheck disable=SC2143 - if [ -z "$(mc ls "storage/isos/cache/$distro" | grep "$sha")" ]; then - curl -L --retry 3 --retry-all-errors -o "cache/$sha.iso" "$url" - if [ "$push" == "yes" ]; then - mc put "cache/$sha.iso" "storage/isos/cache/$distro/$sha.iso" - fi - else - echo "Skipping ${d.distro} - cache/$sha.iso" - fi - '') active - )); + url=$(echo var.iso.url | packer console -var-file="${d}" -config-type=hcl2 sources/) + sha=$(echo var.iso.checksum | packer console -var-file="${d}" -config-type=hcl2 sources/) + distro=$(echo var.distro | packer console -var-file="${d}" -config-type=hcl2 sources/) + echo "$url" "$sha" "$distro" + '') (lib.unique (lib.map distroFile active)) + )) + + '' + } + + list_isos | ${lib.getExe bin} + ''; } diff --git a/nix/cache/cache.go b/nix/cache/cache.go new file mode 100644 index 0000000..bd3f03d --- /dev/null +++ b/nix/cache/cache.go @@ -0,0 +1,166 @@ +package main + +import ( + "bufio" + "context" + "io" + "log" + "net/http" + "os" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) { + var contents []string + + // Create a done channel to control the listing + ctx := context.Background() + + // List all objects from bucket + for object := range minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ + Recursive: true, + }) { + if object.Err != nil { + return nil, object.Err + } + contents = append(contents, object.Key) + } + + return contents, nil +} + +type IsoInfo struct { + Url string + Hash string + Distro string +} + +func processStdin() []IsoInfo { + var isos []IsoInfo + scanner := bufio.NewScanner(os.Stdin) + + for scanner.Scan() { + parts := strings.Split(scanner.Text(), " ") + if len(parts) == 3 { + iso := IsoInfo{ + Url: parts[0], + Hash: parts[1], + Distro: parts[2], + } + isos = append(isos, iso) + } + } + + if err := scanner.Err(); err != nil { + log.Println("Error reading from stdin:", err) + } + + return isos +} + +func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error { + // Create the destination file path + destPath := "cache/" + iso.Hash + ".iso" + // Create cache directory if it doesn't exist + err := os.MkdirAll("cache", 0755) + if err != nil { + return err + } + + log.Println("Downloading " + iso.Url) + // Download the ISO file + resp, err := http.Get(iso.Url) + if err != nil { + return err + } + defer resp.Body.Close() + + // Create the destination file + destFile, err := os.Create(destPath) + if err != nil { + return err + } + defer destFile.Close() + + // Copy the downloaded content to the file + _, err = io.Copy(destFile, resp.Body) + if err != nil { + return err + } + + // Reopen file for uploading + uploadFile, err := os.Open(destPath) + if err != nil { + return err + } + defer uploadFile.Close() + + // Get file info for content-length + fileInfo, err := uploadFile.Stat() + if err != nil { + return err + } + + log.Println("Uploading " + destPath) + uploadPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" + // Upload the file to MinIO + _, err = minioClient.PutObject(context.Background(), "isos", uploadPath, uploadFile, fileInfo.Size(), minio.PutObjectOptions{ + ContentType: "application/x-iso9660-image", + }) + if err != nil { + return err + } + + return nil +} + +func main() { + endpoint := os.Getenv("STORAGE_URL") + log.Println("Using endpoint:", endpoint) + accessKeyID := "root" + secretAccessKey := os.Getenv("MINIO_SECRET") + if secretAccessKey == "" { + log.Fatalln("MINIO_SECRET environment variable not set") + } + useSSL := false + + // Initialize minio client + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + log.Fatalln(err) + } + + // List contents of the "isos" bucket + contents, err := listBucketContents(minioClient, "isos") + if err != nil { + log.Fatalln(err) + } + + isos := processStdin() + + for _, iso := range isos { + // Check if ISO exists in Minio + isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" + exists := false + + for _, item := range contents { + if item == isoPath { + exists = true + break + } + } + + if !exists { + if err := downloadAndUploadIso(minioClient, iso); err != nil { + log.Println("Error downloading/uploading ISO:", err) + os.Exit(1) + } + } + } +} diff --git a/nix/cache/go.mod b/nix/cache/go.mod new file mode 100644 index 0000000..864b321 --- /dev/null +++ b/nix/cache/go.mod @@ -0,0 +1,23 @@ +module main + +go 1.22 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.82 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/schollz/progressbar/v3 v3.17.1 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect +) diff --git a/nix/cache/go.sum b/nix/cache/go.sum new file mode 100644 index 0000000..cc83aef --- /dev/null +++ b/nix/cache/go.sum @@ -0,0 +1,38 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo= +github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= +github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= diff --git a/nix/continue.nix b/nix/continue.nix index f21627e..e4f3223 100644 --- a/nix/continue.nix +++ b/nix/continue.nix @@ -70,6 +70,7 @@ let variables = general.variables // { COMMAND = "packer build -only=\"hyperv-iso.amd64\" -var-file=\"\${distro}\" -var build=\${BUILD} -var cpus=2 -var memory=4096 \${EXCEPT} sources"; }; + rules = [ { "if" = "$CI_COMMIT_TAG"; } ]; script = [ "packer init sources" "echo $env:COMMAND > build.ps1" diff --git a/sources/inputs.pkr.hcl b/sources/inputs.pkr.hcl index 230548e..2fcc09d 100644 --- a/sources/inputs.pkr.hcl +++ b/sources/inputs.pkr.hcl @@ -127,7 +127,7 @@ locals { version = var.version build = var.build }) - iso_location = "s3::${var.storage_url}/isos/cache/${var.distro}/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" + iso_location = "s3::http://${var.storage_url}/isos/cache/${var.distro}/${var.iso.checksum}.iso?aws_access_key_id=root&aws_access_key_secret=${var.minio_secret}" cpus = var.cpus memory = var.memory