chore: fix zims updater

Zims update script has been slightly mangled since nix-prefetch stopped
working.

Now it is updated to use nix-prefetch-url and no longer pulls from the
Torrent sources. That script exports a regular SHA256 hash and not an
SRI signature, so we now convert that to SRI as a second step in the
pre-fetch pipline

Also adding a cron to run the tool every month on the first, in order to
keep it up to date.

.gitea/workflows/update-zims.yaml

@@ -0,0 +1,51 @@
+name: Update zims pin
+
+"on":
+  schedule:
+    - cron: "0 2 1 * *"  # 0200 on the first of every month
+  workflow_dispatch:
+
+jobs:
+  update-flake-lock:
+    runs-on: nix-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update flake.lock
+        run: nix run .#zim-updater -- --output pkgs/zim/blobs.json
+
+      - name: Create PR if changed
+        env:
+          GITEA_TOKEN: ${{ secrets.KLAATU_TOKEN }}
+          GITEA_URL: https://src.thehellings.com
+          REPO: greg/nixos
+        run: |
+          if git diff --quiet pkgs/zim/blobs.json; then
+            echo "blobs.json unchanged, nothing to do"
+            exit 0
+          fi
+
+          BRANCH="auto/update-zims-$(date +%Y%m%d)"
+          git config user.email "klaatu@thehellings.com"
+          git config user.name "klaatu"
+          git checkout -b "$BRANCH"
+          git add pkgs/zim/blobs.json
+          git commit -m "chore: update zim blobs.json $(date +%Y-%m-%d)"
+
+          # Push branch using token auth
+          git remote set-url origin "https://klaatu:${GITEA_TOKEN}@${GITEA_URL#https://}/${REPO}.git"
+          git push origin "$BRANCH"
+
+          # Create PR via Gitea API
+          curl -s -X POST \
+            -H "Authorization: token ${GITEA_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "${GITEA_URL}/api/v1/repos/${REPO}/pulls" \
+            -d "{
+              \"title\": \"chore: update zims $(date +%Y-%m-%d)\",
+              \"head\": \"$BRANCH\",
+              \"base\": \"main\",
+              \"body\": \"Automated monthly zims update.\\n\\nGenerated by Gitea Actions.\",
+              \"assignees\": [\"greg\"]
+            }"

pkgs/default.nix

@@ -16,9 +16,12 @@ let
     adblock_update = c ./adblock_update.nix { };
     brew = c ./homebrew.nix { };
     create_ssl = c ./create_ssl.nix { };
-    dockerCompat = pkgs.runCommand "docker-compat" {
+    dockerCompat =
-      nativeBuildInputs  = [];
+      pkgs.runCommand "docker-compat"
-    } ''
+        {
+          nativeBuildInputs = [ ];
+        }
+        ''
           mkdir -p $out/bin
           ln -s ${pkgs.podman}/bin/podman $out/bin/docker
         '';
@@ -29,6 +32,7 @@ let
     inject = c ./inject.nix { };
     setup-ssh = c ./setup-ssh { };
     upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
+    zim-updater = c ./zim/updater.nix { };
   };
   x86Linux = {
     qemu-hook = c ./qemu-hook.nix { };

pkgs/zim/updater.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strings"
@@ -17,27 +17,153 @@ import (
 
 const BASE = "https://download.kiwix.org/zim"
 
-func getTypes() []string {
+// ///////////////////////////////////////////////////////////////////////////
-	return []string{
+// ////////////////  THE BLOB ITSELF /////////////////////////////////////////
-		"phet",
+// ///////////////////////////////////////////////////////////////////////////
-		"wikipedia",
+type Blobs struct {
-		"wiktionary",
+	En    Language `json:"en"`
-		"wikiversity",
+	Fr    Language `json:"fr"`
-		"wikisource",
+	Ht    Language `json:"ht"`
-		"wikibooks",
+	dirty bool
-		"gutenberg",
+}
-		"ted",
+
+func (b *Blobs) Populate() {
+	done := make(chan bool)
+	waitFor := 0
+	// Launch self-populating efforts
+	blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
+	for f := range blobType.Fields() {
+		if f.IsExported() {
+			//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
+			waitFor += 1
+			go reflect.Indirect(reflect.ValueOf(b)).
+				FieldByName(f.Name).
+				Addr().
+				Interface().
+				(*Language).
+				Populate(strings.ToLower(f.Name), done)
+		}
+	}
+	// Wait until all languages are completed
+	for d := range done {
+		waitFor -= 1
+		if waitFor == 0 {
+			fmt.Println("Completed waiting for all languages")
+			close(done)
+			break
+		}
+		b.dirty = b.dirty || d
 	}
 }
 
-func getLanguages() []string {
+/////////////////////////////////////////////////////////////////////////////
-	return []string{
+//////////////////////// The Language ///////////////////////////////////////
-		"ht",
+/////////////////////////////////////////////////////////////////////////////
-		"en",
+
-		"fr",
+type Language struct {
+	Gutenberg   *Zim `json:"gutenberg,omitempty"`
+	Phet        *Zim `json:"phet,omitempty"`
+	Ted         *Zim `json:"ted,omitempty"`
+	Wikibooks   *Zim `json:"wikibooks,omitempty"`
+	Wikipedia   *Zim `json:"wikipedia,omitempty"`
+	Wikisource  *Zim `json:"wikisource,omitempty"`
+	Wikiversity *Zim `json:"wikiversity,omitempty"`
+	Wiktionary  *Zim `json:"wiktionary,omitempty"`
+	dirty       bool
+}
+
+func (l *Language) Populate(code string, done chan bool) {
+	childDone := make(chan bool)
+	waitFor := 0
+	languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
+	l.dirty = false
+	for f := range languageType.Fields() {
+		if f.IsExported() {
+			waitFor += 1
+			ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
+			    FieldByName(f.Name)
+			if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
+				go reflect.Indirect(ptrPtrZim).
+				    Addr().
+				    Interface().
+				    (*Zim).
+				    Populate(strings.ToLower(f.Name), code, childDone)
+			} else {
+				// TODO: Figure out how to set a value over the nil pointer
+				// of the field, so we can auto-detect when these are available
+				// in the future
+				//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
+				//ptrPtrZim.Set(reflect.ValueOf(z))
+				//go z.Populate(strings.ToLower(f.Name), code, childDone)
+			}
+		}
+	}
+	// Wait until children are all done
+	for d := range childDone {
+		waitFor -= 1
+		if waitFor == 0 {
+			close(childDone)
+			break
+		}
+		l.dirty = l.dirty || d
+	}
+	// Nil out fields that shouldn't be emitted
+	for f := range languageType.Fields() {
+		if f.IsExported() {
+			ptrZim := reflect.Indirect(reflect.ValueOf(l)).
+			    FieldByName(f.Name)
+			if ptrZim.IsValid() && !ptrZim.IsNil() && reflect.Indirect(ptrZim).Addr().Interface().(*Zim).Hash == "" {
+				ptrZim.Set(reflect.ValueOf(nil))
+			}
+		}
+	}
+
+	//fmt.Printf("%s\tFinished populate\n", code)
+	done <- l.dirty
+}
+
+// ///////////////////////////////////////////////////////////////////////////
+// ////////////////////// A Single Zim ///////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////////////
+type Zim struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Hash    string `json:"hash"`
+	dirty   bool
+}
+
+func (z *Zim) Populate(category string, language string, done chan bool) {
+	//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
+	// Look for a possible Zim file
+	links := getLinks(getPage(category))
+	link, err := getName(links, category, language)
+	if err != nil {
+		fmt.Printf("%s:%s\tNo zim found\n", language, category)
+		done <- false
+		return
+	}
+	//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
+	// Extract name and version
+	re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
+	match := re.FindStringSubmatch(link)
+	if len(match) != 3 {
+		fmt.Printf("%s:%s Matches: %s", language, category, match)
+		os.Exit(1)
+	}
+	// Check if the name and version mismatch
+	if match[1] == z.Name && match[2] == z.Version {
+		fmt.Printf("Skipping existing hash: %s\n", link)
+		done <- false
+	} else {
+		z.Name = match[1]
+		z.Version = match[2]
+		// Fetch the Zim file, if it differs from what we currently have
+		z.UpdateHash(category)
+		done <- true
 	}
 }
 
+// Helper function to fetch the HTML of a given type page
 func getPage(t string) string {
 	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
 	if err != nil {
@@ -48,6 +174,7 @@ func getPage(t string) string {
 	return string(pageBytes)
 }
 
+// Helper function to parse out all the links from the page
 func getLinks(page string) []string {
 	ret := []string{}
 
@@ -60,6 +187,7 @@ func getLinks(page string) []string {
 	return ret
 }
 
+// Helper function to get the most likely link for this particular entry
 func getName(links []string, t, lang string) (string, error) {
 	candidates := []string{}
 	prefix := fmt.Sprintf("%s_%s_all", t, lang)
@@ -79,44 +207,39 @@ func getName(links []string, t, lang string) (string, error) {
 	}
 }
 
-func getHash(ch chan result, file, category, language string) {
+// Helper function to get the hash value from the resulting link
-	// TODO: Only call this if the file doesn't already have a hash
+func (z *Zim) UpdateHash(category string) error {
-	// in the existing file
+	file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)
-	fmt.Printf("Fetching hash for %s\n", file)
+
-	cmd := exec.Command( "nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
+	// First fetch the file into our local Nix store
+	fmt.Printf("Fetching hash: %s\n", file)
+	cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
 	out, err := cmd.Output()
 	if err != nil {
-		fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
+		fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
 		if exitErr, ok := err.(*exec.ExitError); ok {
 			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
 		}
-		ch <- result{category, language, ""}
+		return errors.New("Error fetching file")
-		return
 	}
 	hash := strings.TrimSpace(string(out))
-	fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
+	fmt.Printf("%s: Successfully fetched raw hash\n", file)
-	ch <- result{category, language, hash}
+	fmt.Printf("%s: Hash is: %s\n", file, hash)
-}
 
-func outputIsValid(o map[string]map[string]Zim) bool {
+	// Then, convert the hash to SRI
-	for a := range o {
+	cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
-		for b := range o[a] {
+	out2, err2 := cmd2.Output()
-			if o[a][b].Hash == "" {
+	if err2 != nil {
-				return false
+		fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
 		}
+		return errors.New("Error fetching file")
 	}
-	}
+	z.Hash = strings.TrimSpace(string(out2))
-	return true
+	z.dirty = true
-}
+	fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
-
+	return nil
-type result struct {
-	category, language, hash string
-}
-
-type Zim struct {
-	Name string `json:"name"`
-	Version string `json:"version"`
-	Hash string `json:"hash"`
 }
 
 func main() {
@@ -125,84 +248,25 @@ func main() {
 
 	// Determine the output file path
 	var outputPath string
-	if *outputFile != "" {
 	outputPath = *outputFile
-	} else {
-		// Get the directory where updater.go is located
-		execPath, err := os.Executable()
-		if err != nil {
-			// Fallback to current directory if we can't determine executable path
-			outputPath = "blobs.json"
-		} else {
-			dir := filepath.Dir(execPath)
-			outputPath = filepath.Join(dir, "blobs.json")
-		}
-	}
 
 	fmt.Println("Writing file to ", outputPath)
 
 	// Read existing cache if it exists
-	cached := make(map[string]map[string]Zim)
+	var blobs Blobs
 	if data, err := os.ReadFile(outputPath); err == nil {
-		if err := json.Unmarshal(data, &cached); err != nil {
+		if err := json.Unmarshal(data, &blobs); err != nil {
 			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
 		} else {
 			fmt.Printf("Loaded existing cache from %s\n", outputPath)
 		}
+	} else {
+		fmt.Printf("Error reading file: %s", err)
+		os.Exit(1)
 	}
+	blobs.Populate()
 
-	output := make(map[string]map[string]Zim)
+	ret, err := json.MarshalIndent(&blobs, "", "  ")
-	comms := make(chan result)
-	pendingHashes := 0
-
-	for _, t := range getTypes() {
-		page := getPage(t)
-		links := getLinks(page)
-		for _, lang := range getLanguages() {
-			if file, err := getName(links, t, lang); err == nil {
-				if _, ok := output[lang]; !ok {
-					output[lang] = make(map[string]Zim)
-				}
-
-				// Check if this file already exists in cache with same name
-				if cachedLang, ok := cached[lang]; ok {
-					if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
-						// Reuse cached hash
-						fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
-						output[lang][t] = cachedEntry
-						continue
-					}
-				}
-
-				// File is new or name has changed, fetch hash
-				output[lang][t] = Zim{file, ""}
-				pendingHashes++
-				go getHash(comms, file, t, lang)
-			}
-		}
-	}
-
-	// Only wait for results if we actually spawned goroutines
-	if pendingHashes > 0 {
-		hashesReceived := 0
-		for r := range comms {
-			if entry, ok := output[r.language][r.category]; ok {
-				entry.Hash = r.hash
-				output[r.language][r.category] = entry
-			}
-			hashesReceived++
-			if hashesReceived >= pendingHashes {
-				close(comms)
-				break
-			}
-		}
-	}
-
-	// Verify all hashes are present
-	if !outputIsValid(output) {
-		fmt.Println("Warning: Some hashes are missing from the output")
-	}
-	ret, err := json.MarshalIndent(output, "", "  ")
 	if err != nil {
 		fmt.Printf("Error marshaling JSON: %v\n", err)
 		os.Exit(1)