nixos/pkgs/zim/updater.go

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os/exec"
	"regexp"
	"sort"
	"strings"
)

const BASE = "https://download.kiwix.org/zim"

func getTypes() []string {
	return []string{
		"wikipedia",
		"wiktionary",
		"wikiversity",
		"wikisource",
		"wikibooks",
		"gutenberg",
		"phet",
		"ted",
	}
}

func getLanguages() []string {
	return []string{
		"en",
		"fr",
		"ht",
	}
}

func getPage(t string) string {
	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
	if err != nil {
		panic(err)
	}
	defer page.Body.Close()
	pageBytes, _ := io.ReadAll(page.Body)
	return string(pageBytes)
}

func getLinks(page string) []string {
	ret := []string{}

	link_href := regexp.MustCompile(`href="(.*?)"`)
	// Is a list of lists
	links := link_href.FindAllStringSubmatch(page, -1)
	for _, link := range links {
		ret = append(ret, link[len(link)-1])
	}
	return ret
}

func getName(links []string, t, lang string) (string, error) {
	candidates := []string{}
	prefix := fmt.Sprintf("%s_%s_all", t, lang)
	for _, link := range links {
		if strings.HasPrefix(link, prefix) {
			if !strings.HasPrefix(link, "wiki") || strings.Contains(link, "maxi") {
				candidates = append(candidates, link)
			}
		}
	}
	sort.Strings(candidates)
	if len(candidates) > 0 {
		ret := candidates[len(candidates)-1]
		return ret, nil
	} else {
		return "", errors.New("Language item not found")
	}
}

func getHash(ch chan result, file, category, language string) {
	// TODO: Only call this if the file doesn't already have a hash
	// in the existing file
	fmt.Printf("Fetching hash for %s\n", file)
	cmd := exec.Command("nix-prefetch",
        "--option",
        "extra-experimental-features",
        "flakes",
        fmt.Sprintf(`fetchtorrent {
            url="%s/%s/%s.torrent";
            hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
        }`, BASE, category, file),
    )
    out, err := cmd.Output()
    if err != nil {
        panic(err)
    }
	ch <- result{ category, language, strings.TrimSpace(string(out)) }
}

func outputIsValid(o map[string]map[string]Zim) bool {
	for a := range o {
		for b := range o[a] {
			if o[a][b].Hash == "" {
				return false
			}
		}
	}
	return true
}

type result struct {
	category, language, hash string
}

type Zim struct {
	Name string `json:"name"`
	Hash string `json:"hash"`
}

func main() {
	output := make(map[string]map[string]Zim)
	comms := make(chan result)

	for _, t := range getTypes() {
		page := getPage(t)
		links := getLinks(page)
		for _, lang := range getLanguages() {
			if file, err := getName(links, t, lang); err == nil {
				if _, ok := output[lang]; !ok {
					output[lang] = make(map[string]Zim)
				}
				output[lang][t] = Zim{ file, "" }
				go getHash(comms, file, t, lang)
			}
		}
	}

	for r := range comms {
		if entry, ok := output[r.language][r.category]; ok {
			entry.Hash = r.hash
			output[r.language][r.category] = entry
		}
		if outputIsValid(output) {
			close(comms)
			break
		}
	}
	ret, _ := json.Marshal(output)
	fmt.Println(string(ret))
}