Move zims to their own package

Instead of manually downloading the Zim files with a torrent directly,
move them into an actual derivation so it can easily be built into the
resulting images. Also, write an updater script that can automatically
download the files, determine the latest versions for each type and each
language, and generate the hash key values for them

pkgs/zim/updater.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+const BASE = "https://download.kiwix.org/zim"
+
+func getTypes() []string {
+	return []string{
+		"wikipedia",
+		"wiktionary",
+		"wikiversity",
+		"wikisource",
+		"wikibooks",
+		"gutenberg",
+		"phet",
+	}
+}
+
+func getLanguages() []string {
+	return []string{
+		"en",
+		"fr",
+		"ht",
+	}
+}
+
+func getPage(t string) string {
+	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
+	if err != nil {
+		panic(err)
+	}
+	defer page.Body.Close()
+	pageBytes, _ := io.ReadAll(page.Body)
+	return string(pageBytes)
+}
+
+func getLinks(page string) []string {
+	ret := []string{}
+
+	link_href := regexp.MustCompile(`href="(.*?)"`)
+	// Is a list of lists
+	links := link_href.FindAllStringSubmatch(page, -1)
+	for _, link := range links {
+		ret = append(ret, link[len(link)-1])
+	}
+	return ret
+}
+
+func getName(links []string, t, lang string) (string, error) {
+	candidates := []string{}
+	prefix := fmt.Sprintf("%s_%s_all", t, lang)
+	for _, link := range links {
+		if strings.HasPrefix(link, prefix) {
+			if !strings.HasPrefix(link, "wiki") || strings.Contains(link, "maxi") {
+				candidates = append(candidates, link)
+			}
+		}
+	}
+	sort.Strings(candidates)
+	if len(candidates) > 0 {
+		ret := candidates[len(candidates)-1]
+		return ret, nil
+	} else {
+		return "", errors.New("Language item not found")
+	}
+}
+
+func getHash(ch chan result, file, category, language string) {
+	// TODO: Only call this if the file doesn't already have a hash
+	// in the existing file
+	fmt.Printf("Fetching hash for %s\n", file)
+	cmd := exec.Command("nix-prefetch",
+        "--option",
+        "extra-experimental-features",
+        "flakes",
+        fmt.Sprintf(`fetchtorrent {
+            url="%s/%s/%s.torrent";
+            hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
+        }`, BASE, category, file),
+    )
+    out, err := cmd.Output()
+    if err != nil {
+        panic(err)
+    }
+	ch <- result{ category, language, strings.TrimSpace(string(out)) }
+}
+
+func outputIsValid(o map[string]map[string]Zim) bool {
+	for a := range o {
+		for b := range o[a] {
+			if o[a][b].Hash == "" {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+type result struct {
+	category, language, hash string
+}
+
+type Zim struct {
+	Name string `json:"name"`
+	Hash string `json:"hash"`
+}
+
+func main() {
+	output := make(map[string]map[string]Zim)
+	comms := make(chan result)
+
+	for _, t := range getTypes() {
+		page := getPage(t)
+		links := getLinks(page)
+		for _, lang := range getLanguages() {
+			if file, err := getName(links, t, lang); err == nil {
+				if _, ok := output[lang]; !ok {
+					output[lang] = make(map[string]Zim)
+				}
+				output[lang][t] = Zim{ file, "" }
+				go getHash(comms, file, t, lang)
+			}
+		}
+	}
+
+	for r := range comms {
+		if entry, ok := output[r.language][r.category]; ok {
+			entry.Hash = r.hash
+			output[r.language][r.category] = entry
+		}
+		if outputIsValid(output) {
+			close(comms)
+			break
+		}
+	}
+	ret, _ := json.Marshal(output)
+	fmt.Println(string(ret))
+}