diff --git a/pkgs/zim/updater.go b/pkgs/zim/updater.go index 0cebf8c..eb3a349 100644 --- a/pkgs/zim/updater.go +++ b/pkgs/zim/updater.go @@ -3,10 +3,13 @@ package main import ( "encoding/json" "errors" + "flag" "fmt" "io" "net/http" + "os" "os/exec" + "path/filepath" "regexp" "sort" "strings" @@ -16,22 +19,22 @@ const BASE = "https://download.kiwix.org/zim" func getTypes() []string { return []string{ + "phet", "wikipedia", "wiktionary", "wikiversity", "wikisource", "wikibooks", "gutenberg", - "phet", "ted", } } func getLanguages() []string { return []string{ + "ht", "en", "fr", - "ht", } } @@ -87,14 +90,21 @@ func getHash(ch chan result, file, category, language string) { fmt.Sprintf(`fetchtorrent { url="%s/%s/%s.torrent"; hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; - backend="rqbit"; + backend="transmission"; }`, BASE, category, file), ) out, err := cmd.Output() if err != nil { - panic(err) + fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err) + if exitErr, ok := err.(*exec.ExitError); ok { + fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr)) + } + ch <- result{category, language, ""} + return } - ch <- result{category, language, strings.TrimSpace(string(out))} + hash := strings.TrimSpace(string(out)) + fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language) + ch <- result{category, language, hash} } func outputIsValid(o map[string]map[string]Zim) bool { @@ -118,8 +128,38 @@ type Zim struct { } func main() { + outputFile := flag.String("output", "", "Output file path (default: blobs.json in same directory as updater.go)") + flag.Parse() + + // Determine the output file path + var outputPath string + if *outputFile != "" { + outputPath = *outputFile + } else { + // Get the directory where updater.go is located + execPath, err := os.Executable() + if err != nil { + // Fallback to current directory if we can't determine executable path + outputPath = "blobs.json" + } else { + dir := filepath.Dir(execPath) + outputPath = filepath.Join(dir, "blobs.json") + } + } + + // Read existing cache if it exists + cached := make(map[string]map[string]Zim) + if data, err := os.ReadFile(outputPath); err == nil { + if err := json.Unmarshal(data, &cached); err != nil { + fmt.Printf("Warning: could not parse existing cache file: %v\n", err) + } else { + fmt.Printf("Loaded existing cache from %s\n", outputPath) + } + } + output := make(map[string]map[string]Zim) comms := make(chan result) + pendingHashes := 0 for _, t := range getTypes() { page := getPage(t) @@ -129,22 +169,56 @@ func main() { if _, ok := output[lang]; !ok { output[lang] = make(map[string]Zim) } + + // Check if this file already exists in cache with same name + if cachedLang, ok := cached[lang]; ok { + if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file { + // Reuse cached hash + fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang) + output[lang][t] = cachedEntry + continue + } + } + + // File is new or name has changed, fetch hash output[lang][t] = Zim{file, ""} + pendingHashes++ go getHash(comms, file, t, lang) } } } - for r := range comms { - if entry, ok := output[r.language][r.category]; ok { - entry.Hash = r.hash - output[r.language][r.category] = entry - } - if outputIsValid(output) { - close(comms) - break + // Only wait for results if we actually spawned goroutines + if pendingHashes > 0 { + hashesReceived := 0 + for r := range comms { + if entry, ok := output[r.language][r.category]; ok { + entry.Hash = r.hash + output[r.language][r.category] = entry + } + hashesReceived++ + if hashesReceived >= pendingHashes { + close(comms) + break + } } } - ret, _ := json.MarshalIndent(output, " ", "") - fmt.Println(string(ret)) + + // Verify all hashes are present + if !outputIsValid(output) { + fmt.Println("Warning: Some hashes are missing from the output") + } + ret, err := json.MarshalIndent(output, "", " ") + if err != nil { + fmt.Printf("Error marshaling JSON: %v\n", err) + os.Exit(1) + } + + err = os.WriteFile(outputPath, ret, 0644) + if err != nil { + fmt.Printf("Error writing to file %s: %v\n", outputPath, err) + os.Exit(1) + } + + fmt.Printf("Successfully wrote output to %s\n", outputPath) } diff --git a/pkgs/zim/updater.nix b/pkgs/zim/updater.nix index 67be92d..0d4ead9 100644 --- a/pkgs/zim/updater.nix +++ b/pkgs/zim/updater.nix @@ -1,6 +1,7 @@ { writeShellApplication, nix-prefetch, + gcc, go, ... }: @@ -10,10 +11,11 @@ in writeShellApplication { name = "update-zims"; runtimeInputs = [ + gcc go nix-prefetch ]; text = '' - go run ${goscript} + go run ${goscript} -- pkgs/zim/blobs.json ''; }