Support parallel downloads with Go

nix/cache/cache.go

@@ -61,47 +61,59 @@ func processStdin() []IsoInfo {
 	return isos
 }
 
-func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error {
+func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo, ch chan int) {
 	// Create the destination file path
 	destPath := "cache/" + iso.Hash + ".iso"
 	// Create cache directory if it doesn't exist
 	err := os.MkdirAll("cache", 0755)
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 
 	log.Println("Downloading " + iso.Url)
 	// Download the ISO file
 	resp, err := http.Get(iso.Url)
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 	defer resp.Body.Close()
 
 	// Create the destination file
 	destFile, err := os.Create(destPath)
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 	defer destFile.Close()
 
 	// Copy the downloaded content to the file
 	_, err = io.Copy(destFile, resp.Body)
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 
 	// Reopen file for uploading
 	uploadFile, err := os.Open(destPath)
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 	defer uploadFile.Close()
 
 	// Get file info for content-length
 	fileInfo, err := uploadFile.Stat()
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 
 	log.Println("Uploading " + destPath)
@@ -111,10 +123,13 @@ func downloadAndUploadIso(minioClient *minio.Client, iso IsoInfo) error {
 		ContentType: "application/x-iso9660-image",
 	})
 	if err != nil {
-		return err
+		log.Fatalln(err)
+		ch <- 1
+		return
 	}
 
-	return nil
+	log.Println("Done with " + iso.Url)
+	ch <- 1
 }
 
 func main() {
@@ -144,6 +159,8 @@ func main() {
 
 	isos := processStdin()
 
+	downloads := 0
+	dl_counter := make(chan int)
 	for _, iso := range isos {
 		// Check if ISO exists in Minio
 		isoPath := "cache/" + iso.Distro + "/" + iso.Hash + ".iso"
@@ -157,10 +174,16 @@ func main() {
 		}
 
 		if !exists {
-			if err := downloadAndUploadIso(minioClient, iso); err != nil {
-				log.Println("Error downloading/uploading ISO:", err)
-				os.Exit(1)
+			downloads += 1
+			go downloadAndUploadIso(minioClient, iso, dl_counter)
 		}
 	}
+
+	for downloads > 0 {
+		if k := <-dl_counter; k == 1 {
+			downloads -= 1
+		} else {
+			log.Println("Received unknown message on channel: ", k)
+		}
 	}
 }