chore: fix wait-forever bug in updater

chore: update zim pins
chore: fix zims updater
2026-07-27 07:27:34 -05:00 · 2026-07-27 07:19:37 -05:00 · 2026-07-26 21:36:46 -05:00 · 2026-07-26 21:36:46 -05:00
6 changed files with 309 additions and 199 deletions
@@ -0,0 +1,51 @@
+name: Update zims pin
+
+"on":
+  schedule:
+    - cron: "0 2 1 * *"  # 0200 on the first of every month
+  workflow_dispatch:
+
+jobs:
+  update-flake-lock:
+    runs-on: nix-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Update flake.lock
+        run: nix run .#zim-updater -- --output pkgs/zim/blobs.json
+
+      - name: Create PR if changed
+        env:
+          GITEA_TOKEN: ${{ secrets.KLAATU_TOKEN }}
+          GITEA_URL: https://src.thehellings.com
+          REPO: greg/nixos
+        run: |
+          if git diff --quiet pkgs/zim/blobs.json; then
+            echo "blobs.json unchanged, nothing to do"
+            exit 0
+          fi
+
+          BRANCH="auto/update-zims-$(date +%Y%m%d)"
+          git config user.email "klaatu@thehellings.com"
+          git config user.name "klaatu"
+          git checkout -b "$BRANCH"
+          git add pkgs/zim/blobs.json
+          git commit -m "chore: update zim blobs.json $(date +%Y-%m-%d)"
+
+          # Push branch using token auth
+          git remote set-url origin "https://klaatu:${GITEA_TOKEN}@${GITEA_URL#https://}/${REPO}.git"
+          git push origin "$BRANCH"
+
+          # Create PR via Gitea API
+          curl -s -X POST \
+            -H "Authorization: token ${GITEA_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "${GITEA_URL}/api/v1/repos/${REPO}/pulls" \
+            -d "{
+              \"title\": \"chore: update zims $(date +%Y-%m-%d)\",
+              \"head\": \"$BRANCH\",
+              \"base\": \"main\",
+              \"body\": \"Automated monthly zims update.\\n\\nGenerated by Gitea Actions.\",
+              \"assignees\": [\"greg\"]
+            }"
@@ -31,11 +31,11 @@
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
-        "lastModified": 1784438381,
-        "narHash": "sha256-OuKnPzVTpGYaUQp/GjR7NFY8ezVRlR2V2CUxkHkI2UU=",
+        "lastModified": 1783833875,
+        "narHash": "sha256-G+hRtNJ/Nnr6VFMQp2UZdx/ckJDR/RJoK0Fy/yx1/YY=",
        "owner": "nix-community",
        "repo": "buildbot-nix",
-        "rev": "72e12f1a1381321e07279b18f3d401c29871b50e",
+        "rev": "147af587241e2af85399402da60a4f44fdb1e5d7",
        "type": "github"
      },
      "original": {
@@ -53,11 +53,11 @@
        "stable": "stable"
      },
      "locked": {
-        "lastModified": 1784448989,
-        "narHash": "sha256-XC0OkWCiTM91msCp0o7zfM+iBv06mLlCzFxkP8vf+ac=",
+        "lastModified": 1783909498,
+        "narHash": "sha256-T9OfLPLuh1Bf1xojlpWXwooJ6IXapxvb8GM0p3YNy8g=",
        "owner": "zhaofengli",
        "repo": "colmena",
-        "rev": "e5eda626e459f7043868fd2e3b3bf8ce0d1992e6",
+        "rev": "76ba0daa542880b730faec81f4e87efcaa63bc57",
        "type": "github"
      },
      "original": {
@@ -95,11 +95,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1784500460,
-        "narHash": "sha256-UvORnAxTRHax7RG74W8Z2t4GvIkX6AjJ5kk0QlwZomo=",
+        "lastModified": 1784362797,
+        "narHash": "sha256-EP9b9b+OXDxHBPefFwMYCIaLq0fn3UkmrbfzbLUT7kQ=",
        "owner": "lnl7",
        "repo": "nix-darwin",
-        "rev": "57a3171f94705599a2499248ca5758d5eb47c0e0",
+        "rev": "b4cccbd4bc299c1f71ae185b79c3cf99aa82805c",
        "type": "github"
      },
      "original": {
@@ -242,11 +242,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1784913159,
-        "narHash": "sha256-JWq0BfjO4ktpH5USfQNQzdvHpIDT8fSKD5K7LvdMRFs=",
+        "lastModified": 1784407317,
+        "narHash": "sha256-iZrxHToDJWnvt+5LGAtvuQMTy1NZYlNEKbKaVtBJNcc=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "079a3b5d1aa6a719920a51316253b7d6dd22738d",
+        "rev": "39411a8e12a5526d992e65bc7e3dc9a4414d6713",
        "type": "github"
      },
      "original": {
@@ -284,11 +284,11 @@
        "systems": "systems_3"
      },
      "locked": {
-        "lastModified": 1784949919,
-        "narHash": "sha256-itGqjPYHqEUqabXbQb+eHguk1c6rBTRGZS4KImp45kM=",
+        "lastModified": 1784344393,
+        "narHash": "sha256-yAo2ZzSIdeBZg7cOKUpQxwflL+DRYN+1DMObZk2lP2Q=",
        "owner": "Infinidoge",
        "repo": "nix-minecraft",
-        "rev": "a58c1fa23ec0273eb085be1e7874e964aa3cd34f",
+        "rev": "7297d14c52ec8ef39c6aeff2c1818541fd030473",
        "type": "github"
      },
      "original": {
@@ -303,11 +303,11 @@
        "treefmt-nix": "treefmt-nix_2"
      },
      "locked": {
-        "lastModified": 1784621916,
-        "narHash": "sha256-JGjvWpNNrbhiYhx+z9l/iDw1J48TrirfZC6R3yWjmU4=",
+        "lastModified": 1784016977,
+        "narHash": "sha256-TydDba3YD2u15uS0L+PDs7lMqPopnzWggljTKDqOCSw=",
        "owner": "Mic92",
        "repo": "niks3",
-        "rev": "abbc291d372d2d821327516b5f5224de03113204",
+        "rev": "b306808bf381e7e66e33de1e9446a1be0935f4e3",
        "type": "github"
      },
      "original": {
@@ -342,11 +342,11 @@
        "nixpkgs": "nixpkgs_4"
      },
      "locked": {
-        "lastModified": 1784723954,
-        "narHash": "sha256-1CfD8ZUjCkTgjsneLZ/lxCHhgDfqxxE7/GX0MmsgiqA=",
+        "lastModified": 1784310968,
+        "narHash": "sha256-rkSPTePrKqs4dg+i7ZFCq93+HrClac6oSwXX927SVjA=",
        "owner": "nixos",
        "repo": "nixos-hardware",
-        "rev": "a017f5b72210026af5b3ac5949f08d94380a6fbd",
+        "rev": "779c32a00155994c86cde8213a8dd4df139d4355",
        "type": "github"
      },
      "original": {
@@ -388,11 +388,11 @@
    },
    "nixpkgs-lib_2": {
      "locked": {
-        "lastModified": 1785031560,
-        "narHash": "sha256-OmshNvn2vupOFpYinLUu+1Dnpu4n7Q5N3ggGVNHpkUI=",
+        "lastModified": 1783821755,
+        "narHash": "sha256-eMPX9S6MKPyUnaOgeRfrG7OKUiAlc1AlcRinMbSB0WA=",
        "owner": "nix-community",
        "repo": "nixpkgs.lib",
-        "rev": "0e79af5e3d4dcfcd676ab5ba3f95d2e3352e078c",
+        "rev": "228ab8523d81526e57a6ca342e1a919fb6d246a8",
        "type": "github"
      },
      "original": {
@@ -448,11 +448,11 @@
    },
    "nixpkgs_5": {
      "locked": {
-        "lastModified": 1784555310,
-        "narHash": "sha256-/FCliTPgiuV1owejZFNx3Ch9irdvkOfOFl+HHZ+DrtM=",
+        "lastModified": 1783915482,
+        "narHash": "sha256-FmieJB8/OUvNxbkboi7+IGfIuSXY3nF/hZQm8kD0r50=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "421eebfd0ec7bccd4abe826ce62d7e6e83129493",
+        "rev": "6cdc7fc76e8bf7fde9fa43a849fcaaa70e230dee",
        "type": "github"
      },
      "original": {
@@ -464,11 +464,11 @@
    },
    "nixpkgs_6": {
      "locked": {
-        "lastModified": 1784796856,
-        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
+        "lastModified": 1784356753,
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
        "type": "github"
      },
      "original": {
@@ -480,27 +480,27 @@
    },
    "nixpkgs_7": {
      "locked": {
-        "lastModified": 1784564315,
-        "narHash": "sha256-7OpsHHIZFUBLbeHNQ0oC40DZRQjsmGWodXge+OqgPgk=",
+        "lastModified": 1777207419,
+        "narHash": "sha256-V3bmPWAajDiC+1ClDOp55gianW2EyRJSJOyu1RUQibc=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "0c2094806c9e542f31785ef3569ab9e900e3ce9c",
+        "rev": "a7ecea3deccfbdbf22945a89984fcc5a169da8aa",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "0c2094806c9e542f31785ef3569ab9e900e3ce9c",
+        "rev": "a7ecea3deccfbdbf22945a89984fcc5a169da8aa",
        "type": "github"
      }
    },
    "nixunstable": {
      "locked": {
-        "lastModified": 1784796856,
-        "narHash": "sha256-wWFrV5/Qbm+lyt5x20E/bSbfJiGKMo4RCxZV8cl/WZI=",
+        "lastModified": 1784356753,
+        "narHash": "sha256-12KrbMiWLcf8m7pCvAtZh1ZrgF85ZXDXvfR/fWTKy84=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "e2587caef70cea85dd97d7daab492899902dbf5d",
+        "rev": "61b7c44c4073f0b827768aff0049561b5110ea5a",
        "type": "github"
      },
      "original": {
@@ -517,11 +517,11 @@
        "systems": "systems_4"
      },
      "locked": {
-        "lastModified": 1785001306,
-        "narHash": "sha256-xWqmquUtqKXLeDZ1oQUO+rV4sjA9TShIJhLL8M5Bozo=",
+        "lastModified": 1784057377,
+        "narHash": "sha256-yycNej5//EsRbV10moBoh+/63vXEwZD1ZFEiRm6C9rQ=",
        "owner": "nix-community",
        "repo": "nixvim",
-        "rev": "48409beb41e8e28b64e8160ca82d8a93fb628963",
+        "rev": "07180a087e4a00720dc0731cbcd8dec796974381",
        "type": "github"
      },
      "original": {
@@ -537,11 +537,11 @@
        "nixpkgs": "nixpkgs_6"
      },
      "locked": {
-        "lastModified": 1785034246,
-        "narHash": "sha256-/Wul0/XyCTasQ4WYUMHksMffgOOhwJudIzwWT9HpCY4=",
+        "lastModified": 1784417922,
+        "narHash": "sha256-19XZ56wJXArMKxjY25pKXkNp/FrYHGoo+HuQtW6teSM=",
        "owner": "nix-community",
        "repo": "NUR",
-        "rev": "8020cdf42a4c803d7e48bf1375177145deb6cfac",
+        "rev": "2c806d314605495dd7fd75b5950003a062e2b47a",
        "type": "github"
      },
      "original": {
@@ -654,11 +654,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1784369104,
-        "narHash": "sha256-47cxbcZODibHv3rELFQ9vZly0vUNkND/atn/U7HLeb0=",
+        "lastModified": 1780220602,
+        "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=",
        "owner": "numtide",
        "repo": "treefmt-nix",
-        "rev": "df3c0640565d04a0261253cdd89fce78ec50168a",
+        "rev": "db947814a175b7ca6ded66e21383d938df01c227",
        "type": "github"
      },
      "original": {
@@ -693,11 +693,11 @@
        "nixpkgs": "nixpkgs_7"
      },
      "locked": {
-        "lastModified": 1784949765,
-        "narHash": "sha256-MB+0noLfsYMoMsBkyG5CKYZmSk7dSXendvvjfQFUiPw=",
+        "lastModified": 1784343266,
+        "narHash": "sha256-EGkegdTz2n6ESyih8s3dUuPyJQWlYfPp7U41J05g8PY=",
        "owner": "nix-community",
        "repo": "nix-vscode-extensions",
-        "rev": "242a696abba55b45d577b5292e90b597284f5f85",
+        "rev": "472a3e862c76c64ac3ad75a24d332cb5cdd5f1bb",
        "type": "github"
      },
      "original": {
@@ -714,11 +714,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1784642409,
-        "narHash": "sha256-hcbDqFuySAJawljt5r0sKBCJKYnbtGD0T/ZIozH1Dq0=",
+        "lastModified": 1784058842,
+        "narHash": "sha256-3u3tvbCIAid3Mv7RrJx13jusIEQC/HeKYhO/SUSxR3A=",
        "owner": "nix-community",
        "repo": "NixOS-WSL",
-        "rev": "eaeb18da90024448a60eb1ec7132eafa4003404e",
+        "rev": "24c8dc8e0f2170e1a377be24dfadc7d9d21dc1ad",
        "type": "github"
      },
      "original": {
@@ -9,7 +9,7 @@
 {
  imports = [
    ./hardware-configuration.nix
-    top.nix-hardware.nixosModules.framework-11th-gen-intel
+    top.nix-hardware.nixosModules.framework-intel-core-ultra-series1
  ];

  boot = {
@@ -16,12 +16,15 @@ let
    adblock_update = c ./adblock_update.nix { };
    brew = c ./homebrew.nix { };
    create_ssl = c ./create_ssl.nix { };
-    dockerCompat = pkgs.runCommand "docker-compat" {
-      nativeBuildInputs  = [];
-    } ''
-      mkdir -p $out/bin
-      ln -s ${pkgs.podman}/bin/podman $out/bin/docker
-    '';
+    dockerCompat =
+      pkgs.runCommand "docker-compat"
+        {
+          nativeBuildInputs = [ ];
+        }
+        ''
+          mkdir -p $out/bin
+          ln -s ${pkgs.podman}/bin/podman $out/bin/docker
+        '';
    gcc-tune = c ./gcc-tune.nix { };
    #gen-build = c ./gen-build { };
    hms = c ./hms { };
@@ -29,6 +32,7 @@ let
    inject = c ./inject.nix { };
    setup-ssh = c ./setup-ssh { };
    upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
+    zim-updater = c ./zim/updater.nix { };
  };
  x86Linux = {
    qemu-hook = c ./qemu-hook.nix { };
@@ -2,18 +2,18 @@
  "en": {
    "gutenberg": {
      "name": "gutenberg_en_all",
-      "version": "2023-08",
-      "hash": "sha256-OXmdHdsLZcW4nCUQsy7sMpUssS8NV7ztkCgS/nsISuw="
+      "version": "2025-11",
+      "hash": "sha256-AWd8jVVKHKssv9AgrJnryn3Wx0084NE/JmorpgPfryg="
    },
    "phet": {
      "name": "phet_en_all",
-      "version": "2025-03",
-      "hash": "sha256-ARuUzU2o17J2/ZedHc2acXl33dtHCbQJEb72UQUEm1Y="
+      "version": "2026-05",
+      "hash": "sha256-zAAq/X5rjQUiYmjMpBtWP5z3J2ZHJMmIxFKnxLAhOlA="
    },
    "wikibooks": {
      "name": "wikibooks_en_all_maxi",
-      "version": "2025-10",
-      "hash": "sha256-ONBh/ze1Fv2Ffm5b9vDzYS/i2cWSa4pM4MLZnozr2n8="
+      "version": "2026-04",
+      "hash": "sha256-wt7Zr+RkfCsFrOudMCYBADacu6IvPQDkZ6Y0VG2j9hA="
    },
    "wikipedia": {
      "name": "wikipedia_en_all_maxi",
@@ -22,40 +22,40 @@
    },
    "wikisource": {
      "name": "wikisource_en_all_maxi",
-      "version": "2025-11",
-      "hash": "sha256-p1Jio+PMTZVSLIDIOBeMG3acXJg83CwQM7zIWDUyozM="
+      "version": "2026-05",
+      "hash": "sha256-OA4b+U8mxpcX3fst6hrMyctucYTQOlG/b4qZDiVlcf4="
    },
    "wikiversity": {
      "name": "wikiversity_en_all_maxi",
-      "version": "2025-11",
-      "hash": "sha256-IG/gAUdc/vHRrIWGHhw8vMJdLWwqrNbfh+SiclQRIAI="
+      "version": "2026-05",
+      "hash": "sha256-8mZ1CSUcF4QnDIyN0M2KedQ4pcSUCmzHBlOm93MYpCE="
    },
    "wiktionary": {
      "name": "wiktionary_en_all_nopic",
-      "version": "2025-09",
-      "hash": "sha256-Ghcb60qeGaSJtTKQkJoMx/XucX7Lt1XY145lD3gHlMg="
+      "version": "2026-05",
+      "hash": "sha256-Dwiz+viVQt0zb077R9KQRgtUtxG9ixA/DeXAkCdD4rM="
    }
  },
  "fr": {
    "gutenberg": {
      "name": "gutenberg_fr_all",
-      "version": "2025-10",
-      "hash": "sha256-gLHxLXJNwwcxM/mZHtoJ8ojbGQ1fugxgni/+RXccwhU="
+      "version": "2026-01",
+      "hash": "sha256-sGn1TeKwBK0+Er5YOJWq6g0itEm4gIGvKxv/PW0DIao="
    },
    "phet": {
      "name": "phet_fr_all",
-      "version": "2025-03",
-      "hash": "sha256-CSboZNTIowLLhp1u9wkxH8xmu3LvQNx5q493AOYJRIc="
+      "version": "2026-05",
+      "hash": "sha256-bJmchFnaPcofE/hy3ZcRggMT9osHqXwLbwMLJ+cpF28="
    },
    "wikibooks": {
      "name": "wikibooks_fr_all_maxi",
-      "version": "2025-09",
-      "hash": "sha256-KHCc/73L5Bd9iZ47SRV6vpI8fVM1v9hk1TpEvTro2uo="
+      "version": "2026-07",
+      "hash": "sha256-csI+E5UFGi42BLGWWHxQhVr5KSihCFc53KGKwo557Ck="
    },
    "wikipedia": {
      "name": "wikipedia_fr_all_maxi",
-      "version": "2025-06",
-      "hash": "sha256-ve7Mbh96/ObNbV/KVNYZpfLN+HdWlJ7C5LItEdVkRH0="
+      "version": "2026-05",
+      "hash": "sha256-YUAbGzYYr+c2RQqGjHIF9XL1kNsxDgRKnHK1H7/PtDE="
    },
    "wikisource": {
      "name": "wikisource_fr_all_maxi",
@@ -64,25 +64,25 @@
    },
    "wikiversity": {
      "name": "wikiversity_fr_all_maxi",
-      "version": "2025-09",
-      "hash": "sha256-55fdtw/cRezq6nkRp6wuwkukBiqsQKdwh9QWVAgFcBI="
+      "version": "2026-05",
+      "hash": "sha256-nVDHtkWIOJkoiGixnA1zVR1QS58thjc12FJyagK3Ec0="
    },
    "wiktionary": {
      "name": "wiktionary_fr_all_nopic",
-      "version": "2025-11",
-      "hash": "sha256-P0CJptee5rDzccxyRDBzyRLuuFOcaLsJYi6DkBFUfIc="
+      "version": "2026-05",
+      "hash": "sha256-7UXByW2IIL0hec8RPT39jpg5IEGvRMD47hc4Y4vFkJs="
    }
  },
  "ht": {
    "phet": {
      "name": "phet_ht_all",
-      "version": "2025-03",
-      "hash": "sha256-bHrPzE8H7ptrP6r5wPlXSsXNlTiKxJ9KatABC4kwx+s="
+      "version": "2026-05",
+      "hash": "sha256-GiJ1jllhioyMYidQ7+Lcbdd9JN2yf04ZQgnO8XaGAqY="
    },
    "wikipedia": {
      "name": "wikipedia_ht_all_maxi",
-      "version": "2026-04",
-      "hash": "sha256-qUX0pKxIyNsWX4V6kNIsP4Z9nc6TDhwdDR0MmpDOQFs="
+      "version": "2026-07",
+      "hash": "sha256-InS9cMRaIv9ArlKVwKUypz56m4DgSR7Tl6XpSTMzH+o="
    }
  }
 }
@@ -9,7 +9,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strings"
@@ -17,27 +17,144 @@ import (

 const BASE = "https://download.kiwix.org/zim"

-func getTypes() []string {
-	return []string{
-		"phet",
-		"wikipedia",
-		"wiktionary",
-		"wikiversity",
-		"wikisource",
-		"wikibooks",
-		"gutenberg",
-		"ted",
+// ///////////////////////////////////////////////////////////////////////////
+// ////////////////  THE BLOB ITSELF /////////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////////////
+type Blobs struct {
+	En    Language `json:"en"`
+	Fr    Language `json:"fr"`
+	Ht    Language `json:"ht"`
+	dirty bool
+}
+
+func (b *Blobs) Populate() {
+	done := make(chan bool)
+	waitFor := 0
+	// Launch self-populating efforts
+	blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
+	for f := range blobType.Fields() {
+		if f.IsExported() {
+			//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
+			waitFor += 1
+			go reflect.Indirect(reflect.ValueOf(b)).
+				FieldByName(f.Name).
+				Addr().
+				Interface().
+				(*Language).
+				Populate(strings.ToLower(f.Name), done)
+		}
+	}
+	// Wait until all languages are completed
+	for d := range done {
+		waitFor -= 1
+		if waitFor == 0 {
+			fmt.Println("Completed waiting for all languages")
+			close(done)
+			break
+		}
+		b.dirty = b.dirty || d
 	}
 }

-func getLanguages() []string {
-	return []string{
-		"ht",
-		"en",
-		"fr",
+/////////////////////////////////////////////////////////////////////////////
+//////////////////////// The Language ///////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////
+
+type Language struct {
+	Gutenberg   *Zim `json:"gutenberg,omitempty"`
+	Phet        *Zim `json:"phet,omitempty"`
+	Ted         *Zim `json:"ted,omitempty"`
+	Wikibooks   *Zim `json:"wikibooks,omitempty"`
+	Wikipedia   *Zim `json:"wikipedia,omitempty"`
+	Wikisource  *Zim `json:"wikisource,omitempty"`
+	Wikiversity *Zim `json:"wikiversity,omitempty"`
+	Wiktionary  *Zim `json:"wiktionary,omitempty"`
+	dirty       bool
+}
+
+func (l *Language) Populate(code string, done chan bool) {
+	childDone := make(chan bool)
+	waitFor := 0
+	languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
+	l.dirty = false
+	for f := range languageType.Fields() {
+		if f.IsExported() {
+			ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
+			    FieldByName(f.Name)
+			if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
+				waitFor += 1
+				go reflect.Indirect(ptrPtrZim).
+				    Addr().
+				    Interface().
+				    (*Zim).
+				    Populate(strings.ToLower(f.Name), code, childDone)
+			} else {
+				// TODO: Figure out how to set a value over the nil pointer
+				// of the field, so we can auto-detect when these are available
+				// in the future
+				// waitFor += 1
+				//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
+				//ptrPtrZim.Set(reflect.ValueOf(z))
+				//go z.Populate(strings.ToLower(f.Name), code, childDone)
+			}
+		}
+	}
+	// Wait until children are all done
+	for d := range childDone {
+		waitFor -= 1
+		if waitFor == 0 {
+			close(childDone)
+			break
+		}
+		l.dirty = l.dirty || d
+	}
+
+	//fmt.Printf("%s\tFinished populate\n", code)
+	done <- l.dirty
+}
+
+// ///////////////////////////////////////////////////////////////////////////
+// ////////////////////// A Single Zim ///////////////////////////////////////
+// ///////////////////////////////////////////////////////////////////////////
+type Zim struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Hash    string `json:"hash"`
+	dirty   bool
+}
+
+func (z *Zim) Populate(category string, language string, done chan bool) {
+	//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
+	// Look for a possible Zim file
+	links := getLinks(getPage(category))
+	link, err := getName(links, category, language)
+	if err != nil {
+		fmt.Printf("%s:%s\tNo zim found\n", language, category)
+		done <- false
+		return
+	}
+	//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
+	// Extract name and version
+	re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
+	match := re.FindStringSubmatch(link)
+	if len(match) != 3 {
+		fmt.Printf("%s:%s Matches: %s", language, category, match)
+		os.Exit(1)
+	}
+	// Check if the name and version mismatch
+	if match[1] == z.Name && match[2] == z.Version {
+		fmt.Printf("Skipping existing hash: %s\n", link)
+		done <- false
+	} else {
+		z.Name = match[1]
+		z.Version = match[2]
+		// Fetch the Zim file, if it differs from what we currently have
+		z.UpdateHash(category)
+		done <- true
 	}
 }

+// Helper function to fetch the HTML of a given type page
 func getPage(t string) string {
 	page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
 	if err != nil {
@@ -48,6 +165,7 @@ func getPage(t string) string {
 	return string(pageBytes)
 }

+// Helper function to parse out all the links from the page
 func getLinks(page string) []string {
 	ret := []string{}

@@ -60,6 +178,7 @@ func getLinks(page string) []string {
 	return ret
 }

+// Helper function to get the most likely link for this particular entry
 func getName(links []string, t, lang string) (string, error) {
 	candidates := []string{}
 	prefix := fmt.Sprintf("%s_%s_all", t, lang)
@@ -79,44 +198,39 @@ func getName(links []string, t, lang string) (string, error) {
 	}
 }

-func getHash(ch chan result, file, category, language string) {
-	// TODO: Only call this if the file doesn't already have a hash
-	// in the existing file
-	fmt.Printf("Fetching hash for %s\n", file)
-	cmd := exec.Command( "nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
+// Helper function to get the hash value from the resulting link
+func (z *Zim) UpdateHash(category string) error {
+	file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)
+
+	// First fetch the file into our local Nix store
+	fmt.Printf("Fetching hash: %s\n", file)
+	cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
 	out, err := cmd.Output()
 	if err != nil {
-		fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
+		fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
 		if exitErr, ok := err.(*exec.ExitError); ok {
 			fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
 		}
-		ch <- result{category, language, ""}
-		return
+		return errors.New("Error fetching file")
 	}
 	hash := strings.TrimSpace(string(out))
-	fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
-	ch <- result{category, language, hash}
-}
+	fmt.Printf("%s: Successfully fetched raw hash\n", file)
+	fmt.Printf("%s: Hash is: %s\n", file, hash)

-func outputIsValid(o map[string]map[string]Zim) bool {
-	for a := range o {
-		for b := range o[a] {
-			if o[a][b].Hash == "" {
-				return false
-			}
+	// Then, convert the hash to SRI
+	cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
+	out2, err2 := cmd2.Output()
+	if err2 != nil {
+		fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
 		}
+		return errors.New("Error fetching file")
 	}
-	return true
-}
-
-type result struct {
-	category, language, hash string
-}
-
-type Zim struct {
-	Name string `json:"name"`
-	Version string `json:"version"`
-	Hash string `json:"hash"`
+	z.Hash = strings.TrimSpace(string(out2))
+	z.dirty = true
+	fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
+	return nil
 }

 func main() {
@@ -125,90 +239,31 @@ func main() {

 	// Determine the output file path
 	var outputPath string
-	if *outputFile != "" {
-		outputPath = *outputFile
-	} else {
-		// Get the directory where updater.go is located
-		execPath, err := os.Executable()
-		if err != nil {
-			// Fallback to current directory if we can't determine executable path
-			outputPath = "blobs.json"
-		} else {
-			dir := filepath.Dir(execPath)
-			outputPath = filepath.Join(dir, "blobs.json")
-		}
-	}
+	outputPath = *outputFile

 	fmt.Println("Writing file to ", outputPath)

 	// Read existing cache if it exists
-	cached := make(map[string]map[string]Zim)
+	var blobs Blobs
 	if data, err := os.ReadFile(outputPath); err == nil {
-		if err := json.Unmarshal(data, &cached); err != nil {
+		if err := json.Unmarshal(data, &blobs); err != nil {
 			fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
 		} else {
 			fmt.Printf("Loaded existing cache from %s\n", outputPath)
 		}
+	} else {
+		fmt.Printf("Error reading file: %s", err)
+		os.Exit(1)
 	}
+	blobs.Populate()

-	output := make(map[string]map[string]Zim)
-	comms := make(chan result)
-	pendingHashes := 0
-
-	for _, t := range getTypes() {
-		page := getPage(t)
-		links := getLinks(page)
-		for _, lang := range getLanguages() {
-			if file, err := getName(links, t, lang); err == nil {
-				if _, ok := output[lang]; !ok {
-					output[lang] = make(map[string]Zim)
-				}
-
-				// Check if this file already exists in cache with same name
-				if cachedLang, ok := cached[lang]; ok {
-					if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
-						// Reuse cached hash
-						fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
-						output[lang][t] = cachedEntry
-						continue
-					}
-				}
-
-				// File is new or name has changed, fetch hash
-				output[lang][t] = Zim{file, ""}
-				pendingHashes++
-				go getHash(comms, file, t, lang)
-			}
-		}
-	}
-
-	// Only wait for results if we actually spawned goroutines
-	if pendingHashes > 0 {
-		hashesReceived := 0
-		for r := range comms {
-			if entry, ok := output[r.language][r.category]; ok {
-				entry.Hash = r.hash
-				output[r.language][r.category] = entry
-			}
-			hashesReceived++
-			if hashesReceived >= pendingHashes {
-				close(comms)
-				break
-			}
-		}
-	}
-
-	// Verify all hashes are present
-	if !outputIsValid(output) {
-		fmt.Println("Warning: Some hashes are missing from the output")
-	}
-	ret, err := json.MarshalIndent(output, "", "  ")
+	ret, err := json.MarshalIndent(&blobs, "", "  ")
 	if err != nil {
 		fmt.Printf("Error marshaling JSON: %v\n", err)
 		os.Exit(1)
 	}

-	err = os.WriteFile(outputPath, ret, 0644)
+	err = os.WriteFile(outputPath, []byte(fmt.Sprintf("%s\n", ret)), 0644)
 	if err != nil {
 		fmt.Printf("Error writing to file %s: %v\n", outputPath, err)
 		os.Exit(1)
Author	SHA1	Message	Date
Greg Hellings	b9dcd227e8	chore: fix wait-forever bug in updater buildbot/nix-eval Build done. (2 warnings) Details buildbot/nix-build Build done. Details buildbot/nix-effects Build done. Details	2026-07-27 07:27:34 -05:00
Greg Hellings	2d816d50e0	chore: update zim pins	2026-07-27 07:19:37 -05:00
Greg Hellings	5f951c6c12	chore: fix zims updater Zims update script has been slightly mangled since nix-prefetch stopped working. Now it is updated to use nix-prefetch-url and no longer pulls from the Torrent sources. That script exports a regular SHA256 hash and not an SRI signature, so we now convert that to SRI as a second step in the pre-fetch pipline Also adding a cron to run the tool every month on the first, in order to keep it up to date.	2026-07-26 21:36:46 -05:00
Greg Hellings	42efe476db	chore: update framework firmware settings	2026-07-26 21:36:46 -05:00