nixos/pkgs/zim/updater.go

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"reflect"
	"regexp"
	"sort"
	"strings"
)

const BASE = "https://download.kiwix.org/zim"

// ///////////////////////////////////////////////////////////////////////////
// ////////////////  THE BLOB ITSELF /////////////////////////////////////////
// ///////////////////////////////////////////////////////////////////////////
type Blobs struct {
	En    Language `json:"en"`
	Fr    Language `json:"fr"`
	Ht    Language `json:"ht"`
	dirty bool
}

func (b *Blobs) Populate() {
	done := make(chan bool)
	waitFor := 0
	// Launch self-populating efforts
	blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
	for f := range blobType.Fields() {
		if f.IsExported() {
			//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
			waitFor += 1
			go reflect.Indirect(reflect.ValueOf(b)).
				FieldByName(f.Name).
				Addr().
				Interface().
				(*Language).
				Populate(strings.ToLower(f.Name), done)
		}
	}
	// Wait until all languages are completed
	for d := range done {
		waitFor -= 1
		if waitFor == 0 {
			fmt.Println("Completed waiting for all languages")
			close(done)
			break
		}
		b.dirty = b.dirty || d
	}
}

/////////////////////////////////////////////////////////////////////////////
//////////////////////// The Language ///////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////

type Language struct {
	Gutenberg   *Zim `json:"gutenberg,omitempty"`
	Phet        *Zim `json:"phet,omitempty"`
	Ted         *Zim `json:"ted,omitempty"`
	Wikibooks   *Zim `json:"wikibooks,omitempty"`
	Wikipedia   *Zim `json:"wikipedia,omitempty"`
	Wikisource  *Zim `json:"wikisource,omitempty"`
	Wikiversity *Zim `json:"wikiversity,omitempty"`
	Wiktionary  *Zim `json:"wiktionary,omitempty"`
	dirty       bool
}

func (l *Language) Populate(code string, done chan bool) {
	childDone := make(chan bool)
	waitFor := 0
	languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
	l.dirty = false
	for f := range languageType.Fields() {
		if f.IsExported() {
			waitFor += 1
			ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
			    FieldByName(f.Name)
			if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
				go reflect.Indirect(ptrPtrZim).
				    Addr().
				    Interface().
				    (*Zim).
				    Populate(strings.ToLower(f.Name), code, childDone)
			} else {
				// TODO: Figure out how to set a value over the nil pointer
				// of the field, so we can auto-detect when these are available
				// in the future
				//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
				//ptrPtrZim.Set(reflect.ValueOf(z))
				//go z.Populate(strings.ToLower(f.Name), code, childDone)
			}
		}
	}
	// Wait until children are all done
	for d := range childDone {
		waitFor -= 1
		if waitFor == 0 {
			close(childDone)
			break
		}
		l.dirty = l.dirty || d
	}
	// Nil out fields that shouldn't be emitted
	for f := range languageType.Fields() {
		if f.IsExported() {
			ptrZim := reflect.Indirect(reflect.ValueOf(l)).
			    FieldByName(f.Name)
			if ptrZim.IsValid() && !ptrZim.IsNil() && reflect.Indirect(ptrZim).Addr().Interface().(*Zim).Hash == "" {
				ptrZim.Set(reflect.ValueOf(nil))
			}
		}
	}

	//fmt.Printf("%s\tFinished populate\n", code)
	done <- l.dirty
}

// ///////////////////////////////////////////////////////////////////////////
// ////////////////////// A Single Zim ///////////////////////////////////////
// ///////////////////////////////////////////////////////////////////////////
type Zim struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	Hash    string `json:"hash"`
	dirty   bool
}

func (z *Zim) Populate(category string, language string, done chan bool) {
	//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
	// Look for a possible Zim file
	links := getLinks(getPage(category))
	link, err := getName(links, category, language)
	if err != nil {
		fmt.Printf("%s:%s\tNo zim found\n", language, category)
		done <- false
		return
	}
	//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
	// Extract name and version
	re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
	match := re.FindStringSubmatch(link)
	if len(match) != 3 {
		fmt.Printf("%s:%s Matches: %s", language, category, match)
		os.Exit(1)
	}
	// Check if the name and version mismatch
	if match[1] == z.Name && match[2] == z.Version {
		fmt.Printf("Skipping existing hash: %s\n", link)
		done <- false
	} else {
		z.Name = match[1]
		z.Version = match[2]
		// Fetch the Zim file, if it differs from what we currently have
		z.UpdateHash(category)
		done <- true
	}
}

// Helper function to fetch the HTML of a given type page
func getPage(t string) string {
	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
	if err != nil {
		panic(err)
	}
	defer page.Body.Close()
	pageBytes, _ := io.ReadAll(page.Body)
	return string(pageBytes)
}

// Helper function to parse out all the links from the page
func getLinks(page string) []string {
	ret := []string{}

	link_href := regexp.MustCompile(`href="(.*?)"`)
	// Is a list of lists
	links := link_href.FindAllStringSubmatch(page, -1)
	for _, link := range links {
		ret = append(ret, link[len(link)-1])
	}
	return ret
}

// Helper function to get the most likely link for this particular entry
func getName(links []string, t, lang string) (string, error) {
	candidates := []string{}
	prefix := fmt.Sprintf("%s_%s_all", t, lang)
	for _, link := range links {
		if strings.HasPrefix(link, prefix) {
			if !strings.HasPrefix(link, "wiki") || strings.Contains(link, "maxi") {
				candidates = append(candidates, link)
			}
		}
	}
	sort.Strings(candidates)
	if len(candidates) > 0 {
		ret := candidates[len(candidates)-1]
		return ret, nil
	} else {
		return "", errors.New("Language item not found")
	}
}

// Helper function to get the hash value from the resulting link
func (z *Zim) UpdateHash(category string) error {
	file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)

	// First fetch the file into our local Nix store
	fmt.Printf("Fetching hash: %s\n", file)
	cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
	out, err := cmd.Output()
	if err != nil {
		fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
		if exitErr, ok := err.(*exec.ExitError); ok {
			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
		}
		return errors.New("Error fetching file")
	}
	hash := strings.TrimSpace(string(out))
	fmt.Printf("%s: Successfully fetched raw hash\n", file)
	fmt.Printf("%s: Hash is: %s\n", file, hash)

	// Then, convert the hash to SRI
	cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
	out2, err2 := cmd2.Output()
	if err2 != nil {
		fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
		if exitErr, ok := err.(*exec.ExitError); ok {
			fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
		}
		return errors.New("Error fetching file")
	}
	z.Hash = strings.TrimSpace(string(out2))
	z.dirty = true
	fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
	return nil
}

func main() {
	outputFile := flag.String("output", "", "Output file path (default: blobs.json in same directory as updater.go)")
	flag.Parse()

	// Determine the output file path
	var outputPath string
	outputPath = *outputFile

	fmt.Println("Writing file to ", outputPath)

	// Read existing cache if it exists
	var blobs Blobs
	if data, err := os.ReadFile(outputPath); err == nil {
		if err := json.Unmarshal(data, &blobs); err != nil {
			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
		} else {
			fmt.Printf("Loaded existing cache from %s\n", outputPath)
		}
	} else {
		fmt.Printf("Error reading file: %s", err)
		os.Exit(1)
	}
	blobs.Populate()

	ret, err := json.MarshalIndent(&blobs, "", "  ")
	if err != nil {
		fmt.Printf("Error marshaling JSON: %v\n", err)
		os.Exit(1)
	}

	err = os.WriteFile(outputPath, ret, 0644)
	if err != nil {
		fmt.Printf("Error writing to file %s: %v\n", outputPath, err)
		os.Exit(1)
	}

	fmt.Printf("Successfully wrote output to %s\n", outputPath)
	fmt.Println(string(ret))
}