Significant updates to zim update script

No longer silently panic when a command fails
Check that filename has changed and only run when there is a newer
filename
Write output directly to file rather than to stdout

pkgs/zim/updater.go

@@ -3,10 +3,13 @@ package main
 import (
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"
@@ -16,22 +19,22 @@ const BASE = "https://download.kiwix.org/zim"
 
 func getTypes() []string {
 	return []string{
+		"phet",
 		"wikipedia",
 		"wiktionary",
 		"wikiversity",
 		"wikisource",
 		"wikibooks",
 		"gutenberg",
-		"phet",
 		"ted",
 	}
 }
 
 func getLanguages() []string {
 	return []string{
+		"ht",
 		"en",
 		"fr",
-		"ht",
 	}
 }
 
@@ -87,14 +90,21 @@ func getHash(ch chan result, file, category, language string) {
 		fmt.Sprintf(`fetchtorrent {
             url="%s/%s/%s.torrent";
             hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
-            backend="rqbit";
+            backend="transmission";
         }`, BASE, category, file),
 	)
 	out, err := cmd.Output()
 	if err != nil {
-		panic(err)
+		fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
+		}
+		ch <- result{category, language, ""}
+		return
 	}
-	ch <- result{category, language, strings.TrimSpace(string(out))}
+	hash := strings.TrimSpace(string(out))
+	fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
+	ch <- result{category, language, hash}
 }
 
 func outputIsValid(o map[string]map[string]Zim) bool {
@@ -118,8 +128,38 @@ type Zim struct {
 }
 
 func main() {
+	outputFile := flag.String("output", "", "Output file path (default: blobs.json in same directory as updater.go)")
+	flag.Parse()
+
+	// Determine the output file path
+	var outputPath string
+	if *outputFile != "" {
+		outputPath = *outputFile
+	} else {
+		// Get the directory where updater.go is located
+		execPath, err := os.Executable()
+		if err != nil {
+			// Fallback to current directory if we can't determine executable path
+			outputPath = "blobs.json"
+		} else {
+			dir := filepath.Dir(execPath)
+			outputPath = filepath.Join(dir, "blobs.json")
+		}
+	}
+
+	// Read existing cache if it exists
+	cached := make(map[string]map[string]Zim)
+	if data, err := os.ReadFile(outputPath); err == nil {
+		if err := json.Unmarshal(data, &cached); err != nil {
+			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
+		} else {
+			fmt.Printf("Loaded existing cache from %s\n", outputPath)
+		}
+	}
+
 	output := make(map[string]map[string]Zim)
 	comms := make(chan result)
+	pendingHashes := 0
 
 	for _, t := range getTypes() {
 		page := getPage(t)
@@ -129,22 +169,56 @@ func main() {
 				if _, ok := output[lang]; !ok {
 					output[lang] = make(map[string]Zim)
 				}
+
+				// Check if this file already exists in cache with same name
+				if cachedLang, ok := cached[lang]; ok {
+					if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
+						// Reuse cached hash
+						fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
+						output[lang][t] = cachedEntry
+						continue
+					}
+				}
+
+				// File is new or name has changed, fetch hash
 				output[lang][t] = Zim{file, ""}
+				pendingHashes++
 				go getHash(comms, file, t, lang)
 			}
 		}
 	}
 
-	for r := range comms {
-		if entry, ok := output[r.language][r.category]; ok {
-			entry.Hash = r.hash
-			output[r.language][r.category] = entry
-		}
-		if outputIsValid(output) {
-			close(comms)
-			break
+	// Only wait for results if we actually spawned goroutines
+	if pendingHashes > 0 {
+		hashesReceived := 0
+		for r := range comms {
+			if entry, ok := output[r.language][r.category]; ok {
+				entry.Hash = r.hash
+				output[r.language][r.category] = entry
+			}
+			hashesReceived++
+			if hashesReceived >= pendingHashes {
+				close(comms)
+				break
+			}
 		}
 	}
-	ret, _ := json.MarshalIndent(output, "  ", "")
-	fmt.Println(string(ret))
+
+	// Verify all hashes are present
+	if !outputIsValid(output) {
+		fmt.Println("Warning: Some hashes are missing from the output")
+	}
+	ret, err := json.MarshalIndent(output, "", "  ")
+	if err != nil {
+		fmt.Printf("Error marshaling JSON: %v\n", err)
+		os.Exit(1)
+	}
+
+	err = os.WriteFile(outputPath, ret, 0644)
+	if err != nil {
+		fmt.Printf("Error writing to file %s: %v\n", outputPath, err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("Successfully wrote output to %s\n", outputPath)
 }

pkgs/zim/updater.nix

@@ -1,6 +1,7 @@
 {
   writeShellApplication,
   nix-prefetch,
+  gcc,
   go,
   ...
 }:
@@ -10,10 +11,11 @@ in
 writeShellApplication {
   name = "update-zims";
   runtimeInputs = [
+    gcc
     go
     nix-prefetch
   ];
   text = ''
-    go run ${goscript}
+    go run ${goscript} -- pkgs/zim/blobs.json
   '';
 }