chore: fix wait-forever bug in updater

Zims update script has been slightly mangled since nix-prefetch stopped
working.

Now it is updated to use nix-prefetch-url and no longer pulls from the
Torrent sources. That script exports a regular SHA256 hash and not an
SRI signature, so we now convert that to SRI as a second step in the
pre-fetch pipline

Also adding a cron to run the tool every month on the first, in order to
keep it up to date.

.gitea/workflows/update-zims.yaml

@@ -0,0 +1,51 @@
+name: Update zims pin
+
+"on":
+  schedule:
+    - cron: "0 2 1 * *"  # 0200 on the first of every month
+  workflow_dispatch:
+
+jobs:
+  update-flake-lock:
+    runs-on: nix-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update flake.lock
+        run: nix run .#zim-updater -- --output pkgs/zim/blobs.json
+
+      - name: Create PR if changed
+        env:
+          GITEA_TOKEN: ${{ secrets.KLAATU_TOKEN }}
+          GITEA_URL: https://src.thehellings.com
+          REPO: greg/nixos
+        run: |
+          if git diff --quiet pkgs/zim/blobs.json; then
+            echo "blobs.json unchanged, nothing to do"
+            exit 0
+          fi
+
+          BRANCH="auto/update-zims-$(date +%Y%m%d)"
+          git config user.email "klaatu@thehellings.com"
+          git config user.name "klaatu"
+          git checkout -b "$BRANCH"
+          git add pkgs/zim/blobs.json
+          git commit -m "chore: update zim blobs.json $(date +%Y-%m-%d)"
+
+          # Push branch using token auth
+          git remote set-url origin "https://klaatu:${GITEA_TOKEN}@${GITEA_URL#https://}/${REPO}.git"
+          git push origin "$BRANCH"
+
+          # Create PR via Gitea API
+          curl -s -X POST \
+            -H "Authorization: token ${GITEA_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "${GITEA_URL}/api/v1/repos/${REPO}/pulls" \
+            -d "{
+              \"title\": \"chore: update zims $(date +%Y-%m-%d)\",
+              \"head\": \"$BRANCH\",
+              \"base\": \"main\",
+              \"body\": \"Automated monthly zims update.\\n\\nGenerated by Gitea Actions.\",
+              \"assignees\": [\"greg\"]
+            }"

hosts/unstable/exodus/default.nix

@@ -9,7 +9,7 @@
 {
   imports = [
     ./hardware-configuration.nix
-    top.nix-hardware.nixosModules.framework-11th-gen-intel
+    top.nix-hardware.nixosModules.framework-intel-core-ultra-series1
   ];
 
   boot = {

pkgs/default.nix

@@ -16,12 +16,15 @@ let
     adblock_update = c ./adblock_update.nix { };
     brew = c ./homebrew.nix { };
     create_ssl = c ./create_ssl.nix { };
-    dockerCompat = pkgs.runCommand "docker-compat" {
-      nativeBuildInputs  = [];
-    } ''
-      mkdir -p $out/bin
-      ln -s ${pkgs.podman}/bin/podman $out/bin/docker
-    '';
+    dockerCompat =
+      pkgs.runCommand "docker-compat"
+        {
+          nativeBuildInputs = [ ];
+        }
+        ''
+          mkdir -p $out/bin
+          ln -s ${pkgs.podman}/bin/podman $out/bin/docker
+        '';
     gcc-tune = c ./gcc-tune.nix { };
     #gen-build = c ./gen-build { };
     hms = c ./hms { };
@@ -29,6 +32,7 @@ let
     inject = c ./inject.nix { };
     setup-ssh = c ./setup-ssh { };
     upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
+    zim-updater = c ./zim/updater.nix { };
   };
   x86Linux = {
     qemu-hook = c ./qemu-hook.nix { };

pkgs/zim/blobs.json

@@ -2,18 +2,18 @@
   "en": {
     "gutenberg": {
       "name": "gutenberg_en_all",
-      "version": "2023-08",
-      "hash": "sha256-OXmdHdsLZcW4nCUQsy7sMpUssS8NV7ztkCgS/nsISuw="
+      "version": "2025-11",
+      "hash": "sha256-AWd8jVVKHKssv9AgrJnryn3Wx0084NE/JmorpgPfryg="
     },
     "phet": {
       "name": "phet_en_all",
-      "version": "2025-03",
-      "hash": "sha256-ARuUzU2o17J2/ZedHc2acXl33dtHCbQJEb72UQUEm1Y="
+      "version": "2026-05",
+      "hash": "sha256-zAAq/X5rjQUiYmjMpBtWP5z3J2ZHJMmIxFKnxLAhOlA="
     },
     "wikibooks": {
       "name": "wikibooks_en_all_maxi",
-      "version": "2025-10",
-      "hash": "sha256-ONBh/ze1Fv2Ffm5b9vDzYS/i2cWSa4pM4MLZnozr2n8="
+      "version": "2026-04",
+      "hash": "sha256-wt7Zr+RkfCsFrOudMCYBADacu6IvPQDkZ6Y0VG2j9hA="
     },
     "wikipedia": {
       "name": "wikipedia_en_all_maxi",
@@ -22,40 +22,40 @@
     },
     "wikisource": {
       "name": "wikisource_en_all_maxi",
-      "version": "2025-11",
-      "hash": "sha256-p1Jio+PMTZVSLIDIOBeMG3acXJg83CwQM7zIWDUyozM="
+      "version": "2026-05",
+      "hash": "sha256-OA4b+U8mxpcX3fst6hrMyctucYTQOlG/b4qZDiVlcf4="
     },
     "wikiversity": {
       "name": "wikiversity_en_all_maxi",
-      "version": "2025-11",
-      "hash": "sha256-IG/gAUdc/vHRrIWGHhw8vMJdLWwqrNbfh+SiclQRIAI="
+      "version": "2026-05",
+      "hash": "sha256-8mZ1CSUcF4QnDIyN0M2KedQ4pcSUCmzHBlOm93MYpCE="
     },
     "wiktionary": {
       "name": "wiktionary_en_all_nopic",
-      "version": "2025-09",
-      "hash": "sha256-Ghcb60qeGaSJtTKQkJoMx/XucX7Lt1XY145lD3gHlMg="
+      "version": "2026-05",
+      "hash": "sha256-Dwiz+viVQt0zb077R9KQRgtUtxG9ixA/DeXAkCdD4rM="
     }
   },
   "fr": {
     "gutenberg": {
       "name": "gutenberg_fr_all",
-      "version": "2025-10",
-      "hash": "sha256-gLHxLXJNwwcxM/mZHtoJ8ojbGQ1fugxgni/+RXccwhU="
+      "version": "2026-01",
+      "hash": "sha256-sGn1TeKwBK0+Er5YOJWq6g0itEm4gIGvKxv/PW0DIao="
     },
     "phet": {
       "name": "phet_fr_all",
-      "version": "2025-03",
-      "hash": "sha256-CSboZNTIowLLhp1u9wkxH8xmu3LvQNx5q493AOYJRIc="
+      "version": "2026-05",
+      "hash": "sha256-bJmchFnaPcofE/hy3ZcRggMT9osHqXwLbwMLJ+cpF28="
     },
     "wikibooks": {
       "name": "wikibooks_fr_all_maxi",
-      "version": "2025-09",
-      "hash": "sha256-KHCc/73L5Bd9iZ47SRV6vpI8fVM1v9hk1TpEvTro2uo="
+      "version": "2026-07",
+      "hash": "sha256-csI+E5UFGi42BLGWWHxQhVr5KSihCFc53KGKwo557Ck="
     },
     "wikipedia": {
       "name": "wikipedia_fr_all_maxi",
-      "version": "2025-06",
-      "hash": "sha256-ve7Mbh96/ObNbV/KVNYZpfLN+HdWlJ7C5LItEdVkRH0="
+      "version": "2026-05",
+      "hash": "sha256-YUAbGzYYr+c2RQqGjHIF9XL1kNsxDgRKnHK1H7/PtDE="
     },
     "wikisource": {
       "name": "wikisource_fr_all_maxi",
@@ -64,25 +64,25 @@
     },
     "wikiversity": {
       "name": "wikiversity_fr_all_maxi",
-      "version": "2025-09",
-      "hash": "sha256-55fdtw/cRezq6nkRp6wuwkukBiqsQKdwh9QWVAgFcBI="
+      "version": "2026-05",
+      "hash": "sha256-nVDHtkWIOJkoiGixnA1zVR1QS58thjc12FJyagK3Ec0="
     },
     "wiktionary": {
       "name": "wiktionary_fr_all_nopic",
-      "version": "2025-11",
-      "hash": "sha256-P0CJptee5rDzccxyRDBzyRLuuFOcaLsJYi6DkBFUfIc="
+      "version": "2026-05",
+      "hash": "sha256-7UXByW2IIL0hec8RPT39jpg5IEGvRMD47hc4Y4vFkJs="
     }
   },
   "ht": {
     "phet": {
       "name": "phet_ht_all",
-      "version": "2025-03",
-      "hash": "sha256-bHrPzE8H7ptrP6r5wPlXSsXNlTiKxJ9KatABC4kwx+s="
+      "version": "2026-05",
+      "hash": "sha256-GiJ1jllhioyMYidQ7+Lcbdd9JN2yf04ZQgnO8XaGAqY="
     },
     "wikipedia": {
       "name": "wikipedia_ht_all_maxi",
-      "version": "2026-04",
-      "hash": "sha256-qUX0pKxIyNsWX4V6kNIsP4Z9nc6TDhwdDR0MmpDOQFs="
+      "version": "2026-07",
+      "hash": "sha256-InS9cMRaIv9ArlKVwKUypz56m4DgSR7Tl6XpSTMzH+o="
     }
   }
 }

pkgs/zim/updater.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strings"
@@ -17,27 +17,144 @@ import (
 
 const BASE = "https://download.kiwix.org/zim"
 
-func getTypes() []string {
-	return []string{
-		"phet",
-		"wikipedia",
-		"wiktionary",
-		"wikiversity",
-		"wikisource",
-		"wikibooks",
-		"gutenberg",
-		"ted",
+// ///////////////////////////////////////////////////////////////////////////
+// ////////////////  THE BLOB ITSELF /////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////////////
+type Blobs struct {
+	En    Language `json:"en"`
+	Fr    Language `json:"fr"`
+	Ht    Language `json:"ht"`
+	dirty bool
+}
+
+func (b *Blobs) Populate() {
+	done := make(chan bool)
+	waitFor := 0
+	// Launch self-populating efforts
+	blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
+	for f := range blobType.Fields() {
+		if f.IsExported() {
+			//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
+			waitFor += 1
+			go reflect.Indirect(reflect.ValueOf(b)).
+				FieldByName(f.Name).
+				Addr().
+				Interface().
+				(*Language).
+				Populate(strings.ToLower(f.Name), done)
+		}
+	}
+	// Wait until all languages are completed
+	for d := range done {
+		waitFor -= 1
+		if waitFor == 0 {
+			fmt.Println("Completed waiting for all languages")
+			close(done)
+			break
+		}
+		b.dirty = b.dirty || d
 	}
 }
 
-func getLanguages() []string {
-	return []string{
-		"ht",
-		"en",
-		"fr",
+/////////////////////////////////////////////////////////////////////////////
+//////////////////////// The Language ///////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////
+
+type Language struct {
+	Gutenberg   *Zim `json:"gutenberg,omitempty"`
+	Phet        *Zim `json:"phet,omitempty"`
+	Ted         *Zim `json:"ted,omitempty"`
+	Wikibooks   *Zim `json:"wikibooks,omitempty"`
+	Wikipedia   *Zim `json:"wikipedia,omitempty"`
+	Wikisource  *Zim `json:"wikisource,omitempty"`
+	Wikiversity *Zim `json:"wikiversity,omitempty"`
+	Wiktionary  *Zim `json:"wiktionary,omitempty"`
+	dirty       bool
+}
+
+func (l *Language) Populate(code string, done chan bool) {
+	childDone := make(chan bool)
+	waitFor := 0
+	languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
+	l.dirty = false
+	for f := range languageType.Fields() {
+		if f.IsExported() {
+			ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
+			    FieldByName(f.Name)
+			if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
+				waitFor += 1
+				go reflect.Indirect(ptrPtrZim).
+				    Addr().
+				    Interface().
+				    (*Zim).
+				    Populate(strings.ToLower(f.Name), code, childDone)
+			} else {
+				// TODO: Figure out how to set a value over the nil pointer
+				// of the field, so we can auto-detect when these are available
+				// in the future
+				// waitFor += 1
+				//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
+				//ptrPtrZim.Set(reflect.ValueOf(z))
+				//go z.Populate(strings.ToLower(f.Name), code, childDone)
+			}
+		}
+	}
+	// Wait until children are all done
+	for d := range childDone {
+		waitFor -= 1
+		if waitFor == 0 {
+			close(childDone)
+			break
+		}
+		l.dirty = l.dirty || d
+	}
+
+	//fmt.Printf("%s\tFinished populate\n", code)
+	done <- l.dirty
+}
+
+// ///////////////////////////////////////////////////////////////////////////
+// ////////////////////// A Single Zim ///////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////////////
+type Zim struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Hash    string `json:"hash"`
+	dirty   bool
+}
+
+func (z *Zim) Populate(category string, language string, done chan bool) {
+	//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
+	// Look for a possible Zim file
+	links := getLinks(getPage(category))
+	link, err := getName(links, category, language)
+	if err != nil {
+		fmt.Printf("%s:%s\tNo zim found\n", language, category)
+		done <- false
+		return
+	}
+	//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
+	// Extract name and version
+	re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
+	match := re.FindStringSubmatch(link)
+	if len(match) != 3 {
+		fmt.Printf("%s:%s Matches: %s", language, category, match)
+		os.Exit(1)
+	}
+	// Check if the name and version mismatch
+	if match[1] == z.Name && match[2] == z.Version {
+		fmt.Printf("Skipping existing hash: %s\n", link)
+		done <- false
+	} else {
+		z.Name = match[1]
+		z.Version = match[2]
+		// Fetch the Zim file, if it differs from what we currently have
+		z.UpdateHash(category)
+		done <- true
 	}
 }
 
+// Helper function to fetch the HTML of a given type page
 func getPage(t string) string {
 	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
 	if err != nil {
@@ -48,6 +165,7 @@ func getPage(t string) string {
 	return string(pageBytes)
 }
 
+// Helper function to parse out all the links from the page
 func getLinks(page string) []string {
 	ret := []string{}
 
@@ -60,6 +178,7 @@ func getLinks(page string) []string {
 	return ret
 }
 
+// Helper function to get the most likely link for this particular entry
 func getName(links []string, t, lang string) (string, error) {
 	candidates := []string{}
 	prefix := fmt.Sprintf("%s_%s_all", t, lang)
@@ -79,44 +198,39 @@ func getName(links []string, t, lang string) (string, error) {
 	}
 }
 
-func getHash(ch chan result, file, category, language string) {
-	// TODO: Only call this if the file doesn't already have a hash
-	// in the existing file
-	fmt.Printf("Fetching hash for %s\n", file)
-	cmd := exec.Command( "nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
+// Helper function to get the hash value from the resulting link
+func (z *Zim) UpdateHash(category string) error {
+	file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)
+
+	// First fetch the file into our local Nix store
+	fmt.Printf("Fetching hash: %s\n", file)
+	cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
 	out, err := cmd.Output()
 	if err != nil {
-		fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
+		fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
 		if exitErr, ok := err.(*exec.ExitError); ok {
 			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
 		}
-		ch <- result{category, language, ""}
-		return
+		return errors.New("Error fetching file")
 	}
 	hash := strings.TrimSpace(string(out))
-	fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
-	ch <- result{category, language, hash}
-}
+	fmt.Printf("%s: Successfully fetched raw hash\n", file)
+	fmt.Printf("%s: Hash is: %s\n", file, hash)
 
-func outputIsValid(o map[string]map[string]Zim) bool {
-	for a := range o {
-		for b := range o[a] {
-			if o[a][b].Hash == "" {
-				return false
-			}
+	// Then, convert the hash to SRI
+	cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
+	out2, err2 := cmd2.Output()
+	if err2 != nil {
+		fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
 		}
+		return errors.New("Error fetching file")
 	}
-	return true
-}
-
-type result struct {
-	category, language, hash string
-}
-
-type Zim struct {
-	Name string `json:"name"`
-	Version string `json:"version"`
-	Hash string `json:"hash"`
+	z.Hash = strings.TrimSpace(string(out2))
+	z.dirty = true
+	fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
+	return nil
 }
 
 func main() {
@@ -125,90 +239,31 @@ func main() {
 
 	// Determine the output file path
 	var outputPath string
-	if *outputFile != "" {
-		outputPath = *outputFile
-	} else {
-		// Get the directory where updater.go is located
-		execPath, err := os.Executable()
-		if err != nil {
-			// Fallback to current directory if we can't determine executable path
-			outputPath = "blobs.json"
-		} else {
-			dir := filepath.Dir(execPath)
-			outputPath = filepath.Join(dir, "blobs.json")
-		}
-	}
+	outputPath = *outputFile
 
 	fmt.Println("Writing file to ", outputPath)
 
 	// Read existing cache if it exists
-	cached := make(map[string]map[string]Zim)
+	var blobs Blobs
 	if data, err := os.ReadFile(outputPath); err == nil {
-		if err := json.Unmarshal(data, &cached); err != nil {
+		if err := json.Unmarshal(data, &blobs); err != nil {
 			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
 		} else {
 			fmt.Printf("Loaded existing cache from %s\n", outputPath)
 		}
+	} else {
+		fmt.Printf("Error reading file: %s", err)
+		os.Exit(1)
 	}
+	blobs.Populate()
 
-	output := make(map[string]map[string]Zim)
-	comms := make(chan result)
-	pendingHashes := 0
-
-	for _, t := range getTypes() {
-		page := getPage(t)
-		links := getLinks(page)
-		for _, lang := range getLanguages() {
-			if file, err := getName(links, t, lang); err == nil {
-				if _, ok := output[lang]; !ok {
-					output[lang] = make(map[string]Zim)
-				}
-
-				// Check if this file already exists in cache with same name
-				if cachedLang, ok := cached[lang]; ok {
-					if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
-						// Reuse cached hash
-						fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
-						output[lang][t] = cachedEntry
-						continue
-					}
-				}
-
-				// File is new or name has changed, fetch hash
-				output[lang][t] = Zim{file, ""}
-				pendingHashes++
-				go getHash(comms, file, t, lang)
-			}
-		}
-	}
-
-	// Only wait for results if we actually spawned goroutines
-	if pendingHashes > 0 {
-		hashesReceived := 0
-		for r := range comms {
-			if entry, ok := output[r.language][r.category]; ok {
-				entry.Hash = r.hash
-				output[r.language][r.category] = entry
-			}
-			hashesReceived++
-			if hashesReceived >= pendingHashes {
-				close(comms)
-				break
-			}
-		}
-	}
-
-	// Verify all hashes are present
-	if !outputIsValid(output) {
-		fmt.Println("Warning: Some hashes are missing from the output")
-	}
-	ret, err := json.MarshalIndent(output, "", "  ")
+	ret, err := json.MarshalIndent(&blobs, "", "  ")
 	if err != nil {
 		fmt.Printf("Error marshaling JSON: %v\n", err)
 		os.Exit(1)
 	}
 
-	err = os.WriteFile(outputPath, ret, 0644)
+	err = os.WriteFile(outputPath, []byte(fmt.Sprintf("%s\n", ret)), 0644)
 	if err != nil {
 		fmt.Printf("Error writing to file %s: %v\n", outputPath, err)
 		os.Exit(1)