chore: fix zims updater
Zims update script has been slightly mangled since nix-prefetch stopped working. Now it is updated to use nix-prefetch-url and no longer pulls from the Torrent sources. That script exports a regular SHA256 hash and not an SRI signature, so we now convert that to SRI as a second step in the pre-fetch pipline Also adding a cron to run the tool every month on the first, in order to keep it up to date.
This commit is contained in:
Greg Hellings
@@ -0,0 +1,51 @@
|
||||
name: Update zims pin
|
||||
|
||||
"on":
|
||||
schedule:
|
||||
- cron: "0 2 1 * *" # 0200 on the first of every month
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
update-flake-lock:
|
||||
runs-on: nix-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Update flake.lock
|
||||
run: nix run .#zim-updater -- --output pkgs/zim/blobs.json
|
||||
|
||||
- name: Create PR if changed
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.KLAATU_TOKEN }}
|
||||
GITEA_URL: https://src.thehellings.com
|
||||
REPO: greg/nixos
|
||||
run: |
|
||||
if git diff --quiet pkgs/zim/blobs.json; then
|
||||
echo "blobs.json unchanged, nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BRANCH="auto/update-zims-$(date +%Y%m%d)"
|
||||
git config user.email "klaatu@thehellings.com"
|
||||
git config user.name "klaatu"
|
||||
git checkout -b "$BRANCH"
|
||||
git add pkgs/zim/blobs.json
|
||||
git commit -m "chore: update zim blobs.json $(date +%Y-%m-%d)"
|
||||
|
||||
# Push branch using token auth
|
||||
git remote set-url origin "https://klaatu:${GITEA_TOKEN}@${GITEA_URL#https://}/${REPO}.git"
|
||||
git push origin "$BRANCH"
|
||||
|
||||
# Create PR via Gitea API
|
||||
curl -s -X POST \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${GITEA_URL}/api/v1/repos/${REPO}/pulls" \
|
||||
-d "{
|
||||
\"title\": \"chore: update zims $(date +%Y-%m-%d)\",
|
||||
\"head\": \"$BRANCH\",
|
||||
\"base\": \"main\",
|
||||
\"body\": \"Automated monthly zims update.\\n\\nGenerated by Gitea Actions.\",
|
||||
\"assignees\": [\"greg\"]
|
||||
}"
|
||||
+10
-6
@@ -16,12 +16,15 @@ let
|
||||
adblock_update = c ./adblock_update.nix { };
|
||||
brew = c ./homebrew.nix { };
|
||||
create_ssl = c ./create_ssl.nix { };
|
||||
dockerCompat = pkgs.runCommand "docker-compat" {
|
||||
nativeBuildInputs = [];
|
||||
} ''
|
||||
mkdir -p $out/bin
|
||||
ln -s ${pkgs.podman}/bin/podman $out/bin/docker
|
||||
'';
|
||||
dockerCompat =
|
||||
pkgs.runCommand "docker-compat"
|
||||
{
|
||||
nativeBuildInputs = [ ];
|
||||
}
|
||||
''
|
||||
mkdir -p $out/bin
|
||||
ln -s ${pkgs.podman}/bin/podman $out/bin/docker
|
||||
'';
|
||||
gcc-tune = c ./gcc-tune.nix { };
|
||||
#gen-build = c ./gen-build { };
|
||||
hms = c ./hms { };
|
||||
@@ -29,6 +32,7 @@ let
|
||||
inject = c ./inject.nix { };
|
||||
setup-ssh = c ./setup-ssh { };
|
||||
upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
|
||||
zim-updater = c ./zim/updater.nix { };
|
||||
};
|
||||
x86Linux = {
|
||||
qemu-hook = c ./qemu-hook.nix { };
|
||||
|
||||
+175
-111
@@ -9,7 +9,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -17,27 +17,153 @@ import (
|
||||
|
||||
const BASE = "https://download.kiwix.org/zim"
|
||||
|
||||
func getTypes() []string {
|
||||
return []string{
|
||||
"phet",
|
||||
"wikipedia",
|
||||
"wiktionary",
|
||||
"wikiversity",
|
||||
"wikisource",
|
||||
"wikibooks",
|
||||
"gutenberg",
|
||||
"ted",
|
||||
// ///////////////////////////////////////////////////////////////////////////
|
||||
// //////////////// THE BLOB ITSELF /////////////////////////////////////////
|
||||
// ///////////////////////////////////////////////////////////////////////////
|
||||
type Blobs struct {
|
||||
En Language `json:"en"`
|
||||
Fr Language `json:"fr"`
|
||||
Ht Language `json:"ht"`
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func (b *Blobs) Populate() {
|
||||
done := make(chan bool)
|
||||
waitFor := 0
|
||||
// Launch self-populating efforts
|
||||
blobType := reflect.Indirect(reflect.ValueOf(b)).Type()
|
||||
for f := range blobType.Fields() {
|
||||
if f.IsExported() {
|
||||
//fmt.Printf("%s:\tBeginning population efforts\n", f.Name)
|
||||
waitFor += 1
|
||||
go reflect.Indirect(reflect.ValueOf(b)).
|
||||
FieldByName(f.Name).
|
||||
Addr().
|
||||
Interface().
|
||||
(*Language).
|
||||
Populate(strings.ToLower(f.Name), done)
|
||||
}
|
||||
}
|
||||
// Wait until all languages are completed
|
||||
for d := range done {
|
||||
waitFor -= 1
|
||||
if waitFor == 0 {
|
||||
fmt.Println("Completed waiting for all languages")
|
||||
close(done)
|
||||
break
|
||||
}
|
||||
b.dirty = b.dirty || d
|
||||
}
|
||||
}
|
||||
|
||||
func getLanguages() []string {
|
||||
return []string{
|
||||
"ht",
|
||||
"en",
|
||||
"fr",
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////// The Language ///////////////////////////////////////
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
type Language struct {
|
||||
Gutenberg *Zim `json:"gutenberg,omitempty"`
|
||||
Phet *Zim `json:"phet,omitempty"`
|
||||
Ted *Zim `json:"ted,omitempty"`
|
||||
Wikibooks *Zim `json:"wikibooks,omitempty"`
|
||||
Wikipedia *Zim `json:"wikipedia,omitempty"`
|
||||
Wikisource *Zim `json:"wikisource,omitempty"`
|
||||
Wikiversity *Zim `json:"wikiversity,omitempty"`
|
||||
Wiktionary *Zim `json:"wiktionary,omitempty"`
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func (l *Language) Populate(code string, done chan bool) {
|
||||
childDone := make(chan bool)
|
||||
waitFor := 0
|
||||
languageType := reflect.Indirect(reflect.ValueOf(l)).Type()
|
||||
l.dirty = false
|
||||
for f := range languageType.Fields() {
|
||||
if f.IsExported() {
|
||||
waitFor += 1
|
||||
ptrPtrZim := reflect.Indirect(reflect.ValueOf(l)).
|
||||
FieldByName(f.Name)
|
||||
if ptrPtrZim.IsValid() && !ptrPtrZim.IsNil() {
|
||||
go reflect.Indirect(ptrPtrZim).
|
||||
Addr().
|
||||
Interface().
|
||||
(*Zim).
|
||||
Populate(strings.ToLower(f.Name), code, childDone)
|
||||
} else {
|
||||
// TODO: Figure out how to set a value over the nil pointer
|
||||
// of the field, so we can auto-detect when these are available
|
||||
// in the future
|
||||
//z := &Zim{Name: f.Name, Version: "1999-01-01", Hash: ""}
|
||||
//ptrPtrZim.Set(reflect.ValueOf(z))
|
||||
//go z.Populate(strings.ToLower(f.Name), code, childDone)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Wait until children are all done
|
||||
for d := range childDone {
|
||||
waitFor -= 1
|
||||
if waitFor == 0 {
|
||||
close(childDone)
|
||||
break
|
||||
}
|
||||
l.dirty = l.dirty || d
|
||||
}
|
||||
// Nil out fields that shouldn't be emitted
|
||||
for f := range languageType.Fields() {
|
||||
if f.IsExported() {
|
||||
ptrZim := reflect.Indirect(reflect.ValueOf(l)).
|
||||
FieldByName(f.Name)
|
||||
if ptrZim.IsValid() && !ptrZim.IsNil() && reflect.Indirect(ptrZim).Addr().Interface().(*Zim).Hash == "" {
|
||||
ptrZim.Set(reflect.ValueOf(nil))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//fmt.Printf("%s\tFinished populate\n", code)
|
||||
done <- l.dirty
|
||||
}
|
||||
|
||||
// ///////////////////////////////////////////////////////////////////////////
|
||||
// ////////////////////// A Single Zim ///////////////////////////////////////
|
||||
// ///////////////////////////////////////////////////////////////////////////
|
||||
type Zim struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Hash string `json:"hash"`
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func (z *Zim) Populate(category string, language string, done chan bool) {
|
||||
//fmt.Printf("%s:%s\tBeginning populate for\n", language, category)
|
||||
// Look for a possible Zim file
|
||||
links := getLinks(getPage(category))
|
||||
link, err := getName(links, category, language)
|
||||
if err != nil {
|
||||
fmt.Printf("%s:%s\tNo zim found\n", language, category)
|
||||
done <- false
|
||||
return
|
||||
}
|
||||
//fmt.Printf("%s:%s\tFound link: %s\n", category, language, link)
|
||||
// Extract name and version
|
||||
re := regexp.MustCompilePOSIX("^([a-zA-Z_]+)_([0-9-]+)\\.zim$")
|
||||
match := re.FindStringSubmatch(link)
|
||||
if len(match) != 3 {
|
||||
fmt.Printf("%s:%s Matches: %s", language, category, match)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Check if the name and version mismatch
|
||||
if match[1] == z.Name && match[2] == z.Version {
|
||||
fmt.Printf("Skipping existing hash: %s\n", link)
|
||||
done <- false
|
||||
} else {
|
||||
z.Name = match[1]
|
||||
z.Version = match[2]
|
||||
// Fetch the Zim file, if it differs from what we currently have
|
||||
z.UpdateHash(category)
|
||||
done <- true
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to fetch the HTML of a given type page
|
||||
func getPage(t string) string {
|
||||
page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
|
||||
if err != nil {
|
||||
@@ -48,6 +174,7 @@ func getPage(t string) string {
|
||||
return string(pageBytes)
|
||||
}
|
||||
|
||||
// Helper function to parse out all the links from the page
|
||||
func getLinks(page string) []string {
|
||||
ret := []string{}
|
||||
|
||||
@@ -60,6 +187,7 @@ func getLinks(page string) []string {
|
||||
return ret
|
||||
}
|
||||
|
||||
// Helper function to get the most likely link for this particular entry
|
||||
func getName(links []string, t, lang string) (string, error) {
|
||||
candidates := []string{}
|
||||
prefix := fmt.Sprintf("%s_%s_all", t, lang)
|
||||
@@ -79,44 +207,39 @@ func getName(links []string, t, lang string) (string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func getHash(ch chan result, file, category, language string) {
|
||||
// TODO: Only call this if the file doesn't already have a hash
|
||||
// in the existing file
|
||||
fmt.Printf("Fetching hash for %s\n", file)
|
||||
cmd := exec.Command( "nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
|
||||
// Helper function to get the hash value from the resulting link
|
||||
func (z *Zim) UpdateHash(category string) error {
|
||||
file := fmt.Sprintf("%s_%s.zim", z.Name, z.Version)
|
||||
|
||||
// First fetch the file into our local Nix store
|
||||
fmt.Printf("Fetching hash: %s\n", file)
|
||||
cmd := exec.Command("nix-prefetch-url", fmt.Sprintf("%s/%s/%s", BASE, category, file))
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
fmt.Printf("Error fetching hash for %s (category: %s, language: %s): %v\n", file, category, language, err)
|
||||
fmt.Printf("%s: ERROR fetching hash: %v\n", file, err)
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
fmt.Printf("Command stderr: %s\n", string(exitErr.Stderr))
|
||||
}
|
||||
ch <- result{category, language, ""}
|
||||
return
|
||||
return errors.New("Error fetching file")
|
||||
}
|
||||
hash := strings.TrimSpace(string(out))
|
||||
fmt.Printf("Successfully fetched hash for %s (category: %s, language: %s)\n", file, category, language)
|
||||
ch <- result{category, language, hash}
|
||||
}
|
||||
fmt.Printf("%s: Successfully fetched raw hash\n", file)
|
||||
fmt.Printf("%s: Hash is: %s\n", file, hash)
|
||||
|
||||
func outputIsValid(o map[string]map[string]Zim) bool {
|
||||
for a := range o {
|
||||
for b := range o[a] {
|
||||
if o[a][b].Hash == "" {
|
||||
return false
|
||||
}
|
||||
// Then, convert the hash to SRI
|
||||
cmd2 := exec.Command("nix", "hash", "convert", "--to", "sri", hash, "--hash-algo", "sha256")
|
||||
out2, err2 := cmd2.Output()
|
||||
if err2 != nil {
|
||||
fmt.Printf("%s: Error converting hash to SRI: %v\n", file, err)
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
fmt.Printf("%s: Command stderr: %s\n", file, string(exitErr.Stderr))
|
||||
}
|
||||
return errors.New("Error fetching file")
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type result struct {
|
||||
category, language, hash string
|
||||
}
|
||||
|
||||
type Zim struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Hash string `json:"hash"`
|
||||
z.Hash = strings.TrimSpace(string(out2))
|
||||
z.dirty = true
|
||||
fmt.Printf("%s: Successfully convert hash to SRI: %s", file, out2)
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -125,84 +248,25 @@ func main() {
|
||||
|
||||
// Determine the output file path
|
||||
var outputPath string
|
||||
if *outputFile != "" {
|
||||
outputPath = *outputFile
|
||||
} else {
|
||||
// Get the directory where updater.go is located
|
||||
execPath, err := os.Executable()
|
||||
if err != nil {
|
||||
// Fallback to current directory if we can't determine executable path
|
||||
outputPath = "blobs.json"
|
||||
} else {
|
||||
dir := filepath.Dir(execPath)
|
||||
outputPath = filepath.Join(dir, "blobs.json")
|
||||
}
|
||||
}
|
||||
outputPath = *outputFile
|
||||
|
||||
fmt.Println("Writing file to ", outputPath)
|
||||
|
||||
// Read existing cache if it exists
|
||||
cached := make(map[string]map[string]Zim)
|
||||
var blobs Blobs
|
||||
if data, err := os.ReadFile(outputPath); err == nil {
|
||||
if err := json.Unmarshal(data, &cached); err != nil {
|
||||
if err := json.Unmarshal(data, &blobs); err != nil {
|
||||
fmt.Printf("Warning: could not parse existing cache file: %v\n", err)
|
||||
} else {
|
||||
fmt.Printf("Loaded existing cache from %s\n", outputPath)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Error reading file: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
blobs.Populate()
|
||||
|
||||
output := make(map[string]map[string]Zim)
|
||||
comms := make(chan result)
|
||||
pendingHashes := 0
|
||||
|
||||
for _, t := range getTypes() {
|
||||
page := getPage(t)
|
||||
links := getLinks(page)
|
||||
for _, lang := range getLanguages() {
|
||||
if file, err := getName(links, t, lang); err == nil {
|
||||
if _, ok := output[lang]; !ok {
|
||||
output[lang] = make(map[string]Zim)
|
||||
}
|
||||
|
||||
// Check if this file already exists in cache with same name
|
||||
if cachedLang, ok := cached[lang]; ok {
|
||||
if cachedEntry, ok := cachedLang[t]; ok && cachedEntry.Name == file {
|
||||
// Reuse cached hash
|
||||
fmt.Printf("Using cached hash for %s (category: %s, language: %s)\n", file, t, lang)
|
||||
output[lang][t] = cachedEntry
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// File is new or name has changed, fetch hash
|
||||
output[lang][t] = Zim{file, ""}
|
||||
pendingHashes++
|
||||
go getHash(comms, file, t, lang)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only wait for results if we actually spawned goroutines
|
||||
if pendingHashes > 0 {
|
||||
hashesReceived := 0
|
||||
for r := range comms {
|
||||
if entry, ok := output[r.language][r.category]; ok {
|
||||
entry.Hash = r.hash
|
||||
output[r.language][r.category] = entry
|
||||
}
|
||||
hashesReceived++
|
||||
if hashesReceived >= pendingHashes {
|
||||
close(comms)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify all hashes are present
|
||||
if !outputIsValid(output) {
|
||||
fmt.Println("Warning: Some hashes are missing from the output")
|
||||
}
|
||||
ret, err := json.MarshalIndent(output, "", " ")
|
||||
ret, err := json.MarshalIndent(&blobs, "", " ")
|
||||
if err != nil {
|
||||
fmt.Printf("Error marshaling JSON: %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
Reference in New Issue
Block a user