package main import ( "bufio" "context" "io" "log" "net/http" "os" "strings" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func listBucketContents(minioClient *minio.Client, bucketName string) ([]string, error) { var contents []string // Create a done channel to control the listing ctx := context.Background() // List all objects from bucket for object := range minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ Recursive: true, }) { if object.Err != nil { return nil, object.Err } contents = append(contents, object.Key) } return contents, nil } type IsoInfo struct { Url string Hash string Distro string } func processStdin() []IsoInfo { var isos []IsoInfo scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { parts := strings.Split(scanner.Text(), " ") if len(parts) == 3 { iso := IsoInfo{ Url: parts[0], Hash: parts[1], Distro: parts[2], } isos = append(isos, iso) } } if err := scanner.Err(); err != nil { log.Println("Error reading from stdin:", err) } return isos } func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo, ch chan int) { // Create the destination file path destPath := "cache/" + iso.Hash + ".iso" // Create cache directory if it doesn't exist err := os.MkdirAll("cache", 0755) if err != nil { log.Fatalln(err) ch <- 1 return } log.Println("Downloading " + iso.Url) // Download the ISO file resp, err := http.Get(iso.Url) if err != nil { log.Fatalln(err) ch <- 1 return } defer resp.Body.Close() // Create the destination file destFile, err := os.Create(destPath) if err != nil { log.Fatalln(err) ch <- 1 return } defer destFile.Close() // Copy the downloaded content to the file _, err = io.Copy(destFile, resp.Body) if err != nil { log.Fatalln(err) ch <- 1 return } // Reopen file for uploading uploadFile, err := os.Open(destPath) if err != nil { log.Fatalln(err) ch <- 1 return } defer uploadFile.Close() // Get file info for content-length fileInfo, err := uploadFile.Stat() if err != nil { log.Fatalln(err) ch <- 1 return } log.Println("Uploading " + destPath) uploadPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" // Upload the file to MinIO _, err = minioClient.PutObject(context.Background(), "isos", uploadPath, uploadFile, fileInfo.Size(), minio.PutObjectOptions{ ContentType: "application/x-iso9660-image", }) if err != nil { log.Fatalln(err) ch <- 1 return } log.Println("Done with " + iso.Url) ch <- 1 } func main() { endpoint := os.Getenv("STORAGE_URL") log.Println("Using endpoint:", endpoint) accessKeyID := "root" secretAccessKey := os.Getenv("MINIO_SECRET") if secretAccessKey == "" { log.Fatalln("MINIO_SECRET environment variable not set") } useSSL := false // Initialize minio client minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, }) if err != nil { log.Fatalln(err) } // List contents of the "isos" bucket contents, err := listBucketContents(minioClient, "isos") if err != nil { log.Fatalln(err) } isos := processStdin() downloads := 0 dl_counter := make(chan int) for _, iso := range isos { // Check if ISO exists in Minio isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso" exists := false for _, item := range contents { if item == isoPath { exists = true break } } if !exists { downloads += 1 go downloadAndUploadIso(minioClient, iso, dl_counter) } } for downloads > 0 { if k := <-dl_counter; k == 1 { downloads -= 1 } else { log.Println("Received unknown message on channel: ", k) } } }