From 7510136304e2afbe1c2deb5ef997a351ea92402d Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Sat, 12 Oct 2024 10:35:54 -0500 Subject: [PATCH] Move zims to their own package Instead of manually downloading the Zim files with a torrent directly, move them into an actual derivation so it can easily be built into the resulting images. Also, write an updater script that can automatically download the files, determine the latest versions for each type and each language, and generate the hash key values for them --- pkgs/default.nix | 1 + pkgs/zim.nix | 47 ++++++ pkgs/zim/blobs.json | 382 +++++++++++++++++++++++++++++++++++++++++++ pkgs/zim/updater.go | 148 +++++++++++++++++ pkgs/zim/updater.nix | 19 +++ 5 files changed, 597 insertions(+) create mode 100644 pkgs/zim.nix create mode 100644 pkgs/zim/blobs.json create mode 100644 pkgs/zim/updater.go create mode 100644 pkgs/zim/updater.nix diff --git a/pkgs/default.nix b/pkgs/default.nix index 89c9d73..33a175a 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -16,5 +16,6 @@ in hms = c ./hms { }; setup-ssh = c ./setup-ssh { }; upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { }; + zim = c ./zim.nix { }; }; } diff --git a/pkgs/zim.nix b/pkgs/zim.nix new file mode 100644 index 0000000..cf8ac19 --- /dev/null +++ b/pkgs/zim.nix @@ -0,0 +1,47 @@ +{ + callPackage, + fetchtorrent, + lib, + stdenv, + ... +}: +let + inherit (lib.attrsets) mapAttrs mapAttrsRecursiveCond; + # Converts the inputs from the file into a fetchurl command + zim = + type: val: + fetchtorrent { + inherit (val) hash; + url = "https://download.kiwix.org/zim/${type}/${val.name}.torrent"; + curlOptsList = [ "-L" ]; + }; + # Uses the function above to convert the JSON into a structure of fetchurl derivations + zims = mapAttrs (_: types: (mapAttrs (type: hash: zim type hash) types)) ( + builtins.fromJSON (builtins.readFile ./zim/blobs.json) + ); + # Turns the fetchurl derivations into a list of strings, copying the results into a final output + copies = builtins.concatStringsSep "\n" ( + lib.attrsets.collect builtins.isString ( + mapAttrsRecursiveCond (as: !(lib.attrsets.isDerivation as)) # don't recurse derivations + (_: v: "cp ${v} $out/") # copy derivation to destination + zims + ) + ); +in +stdenv.mkDerivation { + name = "zims"; + version = "2024-10"; + + passthru = { + inherit zims; + updater = (callPackage ./zim/updater.nix { }); + }; + + phases = [ "installPhase" ]; + + # Collect all the zim files into a single folder derivation + installPhase = '' + mkdir -p $out + echo "${copies}" > $out/derp.txt + ''; +} diff --git a/pkgs/zim/blobs.json b/pkgs/zim/blobs.json new file mode 100644 index 0000000..827ca87 --- /dev/null +++ b/pkgs/zim/blobs.json @@ -0,0 +1,382 @@ +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +{ + "en": { + "wikipedia": { + "name": "wikipedia_en_all_maxi_2024-01.zim", + "hash": "" + }, + "wiktionary": { + "name": "wiktionary_en_all_nopic_2024-05.zim", + "hash": "" + }, + "wikiversity": { + "name": "wikiversity_en_all_maxi_2021-03.zim", + "hash": "" + }, + "wikibooks": { + "name": "wikibooks_en_all_maxi_2021-03.zim", + "hash": "" + }, + "wikisource": { + "name": "wikisource_en_all_maxi_2022-09.zim", + "hash": "" + }, + "phet": { + "name": "phet_en_all_2023-04.zim", + "hash": "" + }, + "gutenberg": { + "name": "gutenberg_en_all_2023-08.zim", + "hash": "" + } + }, + "fr": { + "wikipedia": { + "name": "wikipedia_fr_all_maxi_2024-05.zim", + "hash": "" + }, + "wiktionary": { + "name": "wiktionary_fr_all_nopic_2024-06.zim", + "hash": "" + }, + "wikiversity": { + "name": "wikiversity_fr_all_maxi_2021-02.zim", + "hash": "" + }, + "wikibooks": { + "name": "wikibooks_fr_all_maxi_2024-06.zim", + "hash": "" + }, + "wikisource": { + "name": "wikisource_fr_all_maxi_2022-04.zim", + "hash": "" + }, + "phet": { + "name": "phet_fr_all_2023-04.zim", + "hash": "" + }, + "gutenberg": { + "name": "gutenberg_fr_all_2023-08.zim", + "hash": "" + } + }, + "ht": { + "wikipedia": { + "name": "wikipedia_ht_all_maxi_2024-06.zim", + "hash": "" + }, + "wikisource": { + "name": "wikisource_ht_all_maxi_2019-03.zim", + "hash": "" + }, + "phet": { + "name": "phet_ht_all_2023-04.zim", + "hash": "" + } + } +} diff --git a/pkgs/zim/updater.go b/pkgs/zim/updater.go new file mode 100644 index 0000000..9eb4688 --- /dev/null +++ b/pkgs/zim/updater.go @@ -0,0 +1,148 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os/exec" + "regexp" + "sort" + "strings" +) + +const BASE = "https://download.kiwix.org/zim" + +func getTypes() []string { + return []string{ + "wikipedia", + "wiktionary", + "wikiversity", + "wikisource", + "wikibooks", + "gutenberg", + "phet", + } +} + +func getLanguages() []string { + return []string{ + "en", + "fr", + "ht", + } +} + +func getPage(t string) string { + page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t)) + if err != nil { + panic(err) + } + defer page.Body.Close() + pageBytes, _ := io.ReadAll(page.Body) + return string(pageBytes) +} + +func getLinks(page string) []string { + ret := []string{} + + link_href := regexp.MustCompile(`href="(.*?)"`) + // Is a list of lists + links := link_href.FindAllStringSubmatch(page, -1) + for _, link := range links { + ret = append(ret, link[len(link)-1]) + } + return ret +} + +func getName(links []string, t, lang string) (string, error) { + candidates := []string{} + prefix := fmt.Sprintf("%s_%s_all", t, lang) + for _, link := range links { + if strings.HasPrefix(link, prefix) { + if !strings.HasPrefix(link, "wiki") || strings.Contains(link, "maxi") { + candidates = append(candidates, link) + } + } + } + sort.Strings(candidates) + if len(candidates) > 0 { + ret := candidates[len(candidates)-1] + return ret, nil + } else { + return "", errors.New("Language item not found") + } +} + +func getHash(ch chan result, file, category, language string) { + // TODO: Only call this if the file doesn't already have a hash + // in the existing file + fmt.Printf("Fetching hash for %s\n", file) + cmd := exec.Command("nix-prefetch", + "--option", + "extra-experimental-features", + "flakes", + fmt.Sprintf(`fetchtorrent { + url="%s/%s/%s.torrent"; + hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }`, BASE, category, file), + ) + out, err := cmd.Output() + if err != nil { + panic(err) + } + ch <- result{ category, language, strings.TrimSpace(string(out)) } +} + +func outputIsValid(o map[string]map[string]Zim) bool { + for a := range o { + for b := range o[a] { + if o[a][b].Hash == "" { + return false + } + } + } + return true +} + +type result struct { + category, language, hash string +} + +type Zim struct { + Name string `json:"name"` + Hash string `json:"hash"` +} + +func main() { + output := make(map[string]map[string]Zim) + comms := make(chan result) + + for _, t := range getTypes() { + page := getPage(t) + links := getLinks(page) + for _, lang := range getLanguages() { + if file, err := getName(links, t, lang); err == nil { + if _, ok := output[lang]; !ok { + output[lang] = make(map[string]Zim) + } + output[lang][t] = Zim{ file, "" } + go getHash(comms, file, t, lang) + } + } + } + + for r := range comms { + if entry, ok := output[r.language][r.category]; ok { + entry.Hash = r.hash + output[r.language][r.category] = entry + } + if outputIsValid(output) { + close(comms) + break + } + } + ret, _ := json.Marshal(output) + fmt.Println(string(ret)) +} diff --git a/pkgs/zim/updater.nix b/pkgs/zim/updater.nix new file mode 100644 index 0000000..67be92d --- /dev/null +++ b/pkgs/zim/updater.nix @@ -0,0 +1,19 @@ +{ + writeShellApplication, + nix-prefetch, + go, + ... +}: +let + goscript = ./updater.go; +in +writeShellApplication { + name = "update-zims"; + runtimeInputs = [ + go + nix-prefetch + ]; + text = '' + go run ${goscript} + ''; +}