diff --git a/pkgs/default.nix b/pkgs/default.nix index 89c9d73..33a175a 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -16,5 +16,6 @@ in hms = c ./hms { }; setup-ssh = c ./setup-ssh { }; upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { }; + zim = c ./zim.nix { }; }; } diff --git a/pkgs/zim.nix b/pkgs/zim.nix new file mode 100644 index 0000000..cf8ac19 --- /dev/null +++ b/pkgs/zim.nix @@ -0,0 +1,47 @@ +{ + callPackage, + fetchtorrent, + lib, + stdenv, + ... +}: +let + inherit (lib.attrsets) mapAttrs mapAttrsRecursiveCond; + # Converts the inputs from the file into a fetchurl command + zim = + type: val: + fetchtorrent { + inherit (val) hash; + url = "https://download.kiwix.org/zim/${type}/${val.name}.torrent"; + curlOptsList = [ "-L" ]; + }; + # Uses the function above to convert the JSON into a structure of fetchurl derivations + zims = mapAttrs (_: types: (mapAttrs (type: hash: zim type hash) types)) ( + builtins.fromJSON (builtins.readFile ./zim/blobs.json) + ); + # Turns the fetchurl derivations into a list of strings, copying the results into a final output + copies = builtins.concatStringsSep "\n" ( + lib.attrsets.collect builtins.isString ( + mapAttrsRecursiveCond (as: !(lib.attrsets.isDerivation as)) # don't recurse derivations + (_: v: "cp ${v} $out/") # copy derivation to destination + zims + ) + ); +in +stdenv.mkDerivation { + name = "zims"; + version = "2024-10"; + + passthru = { + inherit zims; + updater = (callPackage ./zim/updater.nix { }); + }; + + phases = [ "installPhase" ]; + + # Collect all the zim files into a single folder derivation + installPhase = '' + mkdir -p $out + echo "${copies}" > $out/derp.txt + ''; +} diff --git a/pkgs/zim/blobs.json b/pkgs/zim/blobs.json new file mode 100644 index 0000000..827ca87 --- /dev/null +++ b/pkgs/zim/blobs.json @@ -0,0 +1,382 @@ +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +error: + … + + at «none»:0: (source not available) + + … in the left operand of the update (//) operator + + at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46: + + 2| + 3| let prelude = with prelude; import ./lib.nix // { + | ^ + 4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ] + + (stack trace truncated; use '--show-trace' to show the full trace) + + error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override + +{ + "en": { + "wikipedia": { + "name": "wikipedia_en_all_maxi_2024-01.zim", + "hash": "" + }, + "wiktionary": { + "name": "wiktionary_en_all_nopic_2024-05.zim", + "hash": "" + }, + "wikiversity": { + "name": "wikiversity_en_all_maxi_2021-03.zim", + "hash": "" + }, + "wikibooks": { + "name": "wikibooks_en_all_maxi_2021-03.zim", + "hash": "" + }, + "wikisource": { + "name": "wikisource_en_all_maxi_2022-09.zim", + "hash": "" + }, + "phet": { + "name": "phet_en_all_2023-04.zim", + "hash": "" + }, + "gutenberg": { + "name": "gutenberg_en_all_2023-08.zim", + "hash": "" + } + }, + "fr": { + "wikipedia": { + "name": "wikipedia_fr_all_maxi_2024-05.zim", + "hash": "" + }, + "wiktionary": { + "name": "wiktionary_fr_all_nopic_2024-06.zim", + "hash": "" + }, + "wikiversity": { + "name": "wikiversity_fr_all_maxi_2021-02.zim", + "hash": "" + }, + "wikibooks": { + "name": "wikibooks_fr_all_maxi_2024-06.zim", + "hash": "" + }, + "wikisource": { + "name": "wikisource_fr_all_maxi_2022-04.zim", + "hash": "" + }, + "phet": { + "name": "phet_fr_all_2023-04.zim", + "hash": "" + }, + "gutenberg": { + "name": "gutenberg_fr_all_2023-08.zim", + "hash": "" + } + }, + "ht": { + "wikipedia": { + "name": "wikipedia_ht_all_maxi_2024-06.zim", + "hash": "" + }, + "wikisource": { + "name": "wikisource_ht_all_maxi_2019-03.zim", + "hash": "" + }, + "phet": { + "name": "phet_ht_all_2023-04.zim", + "hash": "" + } + } +} diff --git a/pkgs/zim/updater.go b/pkgs/zim/updater.go new file mode 100644 index 0000000..9eb4688 --- /dev/null +++ b/pkgs/zim/updater.go @@ -0,0 +1,148 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os/exec" + "regexp" + "sort" + "strings" +) + +const BASE = "https://download.kiwix.org/zim" + +func getTypes() []string { + return []string{ + "wikipedia", + "wiktionary", + "wikiversity", + "wikisource", + "wikibooks", + "gutenberg", + "phet", + } +} + +func getLanguages() []string { + return []string{ + "en", + "fr", + "ht", + } +} + +func getPage(t string) string { + page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t)) + if err != nil { + panic(err) + } + defer page.Body.Close() + pageBytes, _ := io.ReadAll(page.Body) + return string(pageBytes) +} + +func getLinks(page string) []string { + ret := []string{} + + link_href := regexp.MustCompile(`href="(.*?)"`) + // Is a list of lists + links := link_href.FindAllStringSubmatch(page, -1) + for _, link := range links { + ret = append(ret, link[len(link)-1]) + } + return ret +} + +func getName(links []string, t, lang string) (string, error) { + candidates := []string{} + prefix := fmt.Sprintf("%s_%s_all", t, lang) + for _, link := range links { + if strings.HasPrefix(link, prefix) { + if !strings.HasPrefix(link, "wiki") || strings.Contains(link, "maxi") { + candidates = append(candidates, link) + } + } + } + sort.Strings(candidates) + if len(candidates) > 0 { + ret := candidates[len(candidates)-1] + return ret, nil + } else { + return "", errors.New("Language item not found") + } +} + +func getHash(ch chan result, file, category, language string) { + // TODO: Only call this if the file doesn't already have a hash + // in the existing file + fmt.Printf("Fetching hash for %s\n", file) + cmd := exec.Command("nix-prefetch", + "--option", + "extra-experimental-features", + "flakes", + fmt.Sprintf(`fetchtorrent { + url="%s/%s/%s.torrent"; + hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }`, BASE, category, file), + ) + out, err := cmd.Output() + if err != nil { + panic(err) + } + ch <- result{ category, language, strings.TrimSpace(string(out)) } +} + +func outputIsValid(o map[string]map[string]Zim) bool { + for a := range o { + for b := range o[a] { + if o[a][b].Hash == "" { + return false + } + } + } + return true +} + +type result struct { + category, language, hash string +} + +type Zim struct { + Name string `json:"name"` + Hash string `json:"hash"` +} + +func main() { + output := make(map[string]map[string]Zim) + comms := make(chan result) + + for _, t := range getTypes() { + page := getPage(t) + links := getLinks(page) + for _, lang := range getLanguages() { + if file, err := getName(links, t, lang); err == nil { + if _, ok := output[lang]; !ok { + output[lang] = make(map[string]Zim) + } + output[lang][t] = Zim{ file, "" } + go getHash(comms, file, t, lang) + } + } + } + + for r := range comms { + if entry, ok := output[r.language][r.category]; ok { + entry.Hash = r.hash + output[r.language][r.category] = entry + } + if outputIsValid(output) { + close(comms) + break + } + } + ret, _ := json.Marshal(output) + fmt.Println(string(ret)) +} diff --git a/pkgs/zim/updater.nix b/pkgs/zim/updater.nix new file mode 100644 index 0000000..67be92d --- /dev/null +++ b/pkgs/zim/updater.nix @@ -0,0 +1,19 @@ +{ + writeShellApplication, + nix-prefetch, + go, + ... +}: +let + goscript = ./updater.go; +in +writeShellApplication { + name = "update-zims"; + runtimeInputs = [ + go + nix-prefetch + ]; + text = '' + go run ${goscript} + ''; +}