Move zims to their own package
Instead of manually downloading the Zim files with a torrent directly, move them into an actual derivation so it can easily be built into the resulting images. Also, write an updater script that can automatically download the files, determine the latest versions for each type and each language, and generate the hash key values for them
This commit is contained in:
Greg Hellings
@@ -16,5 +16,6 @@ in
|
||||
hms = c ./hms { };
|
||||
setup-ssh = c ./setup-ssh { };
|
||||
upgrade-pg-cluster = c ./upgrade-pg-cluster.nix { };
|
||||
zim = c ./zim.nix { };
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
callPackage,
|
||||
fetchtorrent,
|
||||
lib,
|
||||
stdenv,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (lib.attrsets) mapAttrs mapAttrsRecursiveCond;
|
||||
# Converts the inputs from the file into a fetchurl command
|
||||
zim =
|
||||
type: val:
|
||||
fetchtorrent {
|
||||
inherit (val) hash;
|
||||
url = "https://download.kiwix.org/zim/${type}/${val.name}.torrent";
|
||||
curlOptsList = [ "-L" ];
|
||||
};
|
||||
# Uses the function above to convert the JSON into a structure of fetchurl derivations
|
||||
zims = mapAttrs (_: types: (mapAttrs (type: hash: zim type hash) types)) (
|
||||
builtins.fromJSON (builtins.readFile ./zim/blobs.json)
|
||||
);
|
||||
# Turns the fetchurl derivations into a list of strings, copying the results into a final output
|
||||
copies = builtins.concatStringsSep "\n" (
|
||||
lib.attrsets.collect builtins.isString (
|
||||
mapAttrsRecursiveCond (as: !(lib.attrsets.isDerivation as)) # don't recurse derivations
|
||||
(_: v: "cp ${v} $out/") # copy derivation to destination
|
||||
zims
|
||||
)
|
||||
);
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
name = "zims";
|
||||
version = "2024-10";
|
||||
|
||||
passthru = {
|
||||
inherit zims;
|
||||
updater = (callPackage ./zim/updater.nix { });
|
||||
};
|
||||
|
||||
phases = [ "installPhase" ];
|
||||
|
||||
# Collect all the zim files into a single folder derivation
|
||||
installPhase = ''
|
||||
mkdir -p $out
|
||||
echo "${copies}" > $out/derp.txt
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
error:
|
||||
… <borked>
|
||||
|
||||
at «none»:0: (source not available)
|
||||
|
||||
… in the left operand of the update (//) operator
|
||||
|
||||
at /nix/store/3hh72ymqzdmlxzw39zgmadigibrqv7wl-nix-prefetch-0.4.1/lib/nix-prefetch/prelude.nix:3:46:
|
||||
|
||||
2|
|
||||
3| let prelude = with prelude; import ./lib.nix // {
|
||||
| ^
|
||||
4| fetcher = if orig.fetcher == null || elem orig.fetcher.type [ "file" "attr" ]
|
||||
|
||||
(stack trace truncated; use '--show-trace' to show the full trace)
|
||||
|
||||
error: experimental Nix feature 'flakes' is disabled; use '--extra-experimental-features flakes' to override
|
||||
|
||||
{
|
||||
"en": {
|
||||
"wikipedia": {
|
||||
"name": "wikipedia_en_all_maxi_2024-01.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wiktionary": {
|
||||
"name": "wiktionary_en_all_nopic_2024-05.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikiversity": {
|
||||
"name": "wikiversity_en_all_maxi_2021-03.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikibooks": {
|
||||
"name": "wikibooks_en_all_maxi_2021-03.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikisource": {
|
||||
"name": "wikisource_en_all_maxi_2022-09.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"phet": {
|
||||
"name": "phet_en_all_2023-04.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"gutenberg": {
|
||||
"name": "gutenberg_en_all_2023-08.zim",
|
||||
"hash": ""
|
||||
}
|
||||
},
|
||||
"fr": {
|
||||
"wikipedia": {
|
||||
"name": "wikipedia_fr_all_maxi_2024-05.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wiktionary": {
|
||||
"name": "wiktionary_fr_all_nopic_2024-06.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikiversity": {
|
||||
"name": "wikiversity_fr_all_maxi_2021-02.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikibooks": {
|
||||
"name": "wikibooks_fr_all_maxi_2024-06.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikisource": {
|
||||
"name": "wikisource_fr_all_maxi_2022-04.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"phet": {
|
||||
"name": "phet_fr_all_2023-04.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"gutenberg": {
|
||||
"name": "gutenberg_fr_all_2023-08.zim",
|
||||
"hash": ""
|
||||
}
|
||||
},
|
||||
"ht": {
|
||||
"wikipedia": {
|
||||
"name": "wikipedia_ht_all_maxi_2024-06.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"wikisource": {
|
||||
"name": "wikisource_ht_all_maxi_2019-03.zim",
|
||||
"hash": ""
|
||||
},
|
||||
"phet": {
|
||||
"name": "phet_ht_all_2023-04.zim",
|
||||
"hash": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const BASE = "https://download.kiwix.org/zim"
|
||||
|
||||
func getTypes() []string {
|
||||
return []string{
|
||||
"wikipedia",
|
||||
"wiktionary",
|
||||
"wikiversity",
|
||||
"wikisource",
|
||||
"wikibooks",
|
||||
"gutenberg",
|
||||
"phet",
|
||||
}
|
||||
}
|
||||
|
||||
func getLanguages() []string {
|
||||
return []string{
|
||||
"en",
|
||||
"fr",
|
||||
"ht",
|
||||
}
|
||||
}
|
||||
|
||||
func getPage(t string) string {
|
||||
page, err := http.Get(fmt.Sprintf("%s/%s/", BASE, t))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer page.Body.Close()
|
||||
pageBytes, _ := io.ReadAll(page.Body)
|
||||
return string(pageBytes)
|
||||
}
|
||||
|
||||
func getLinks(page string) []string {
|
||||
ret := []string{}
|
||||
|
||||
link_href := regexp.MustCompile(`href="(.*?)"`)
|
||||
// Is a list of lists
|
||||
links := link_href.FindAllStringSubmatch(page, -1)
|
||||
for _, link := range links {
|
||||
ret = append(ret, link[len(link)-1])
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func getName(links []string, t, lang string) (string, error) {
|
||||
candidates := []string{}
|
||||
prefix := fmt.Sprintf("%s_%s_all", t, lang)
|
||||
for _, link := range links {
|
||||
if strings.HasPrefix(link, prefix) {
|
||||
if !strings.HasPrefix(link, "wiki") || strings.Contains(link, "maxi") {
|
||||
candidates = append(candidates, link)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(candidates)
|
||||
if len(candidates) > 0 {
|
||||
ret := candidates[len(candidates)-1]
|
||||
return ret, nil
|
||||
} else {
|
||||
return "", errors.New("Language item not found")
|
||||
}
|
||||
}
|
||||
|
||||
func getHash(ch chan result, file, category, language string) {
|
||||
// TODO: Only call this if the file doesn't already have a hash
|
||||
// in the existing file
|
||||
fmt.Printf("Fetching hash for %s\n", file)
|
||||
cmd := exec.Command("nix-prefetch",
|
||||
"--option",
|
||||
"extra-experimental-features",
|
||||
"flakes",
|
||||
fmt.Sprintf(`fetchtorrent {
|
||||
url="%s/%s/%s.torrent";
|
||||
hash="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
||||
}`, BASE, category, file),
|
||||
)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ch <- result{ category, language, strings.TrimSpace(string(out)) }
|
||||
}
|
||||
|
||||
func outputIsValid(o map[string]map[string]Zim) bool {
|
||||
for a := range o {
|
||||
for b := range o[a] {
|
||||
if o[a][b].Hash == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type result struct {
|
||||
category, language, hash string
|
||||
}
|
||||
|
||||
type Zim struct {
|
||||
Name string `json:"name"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
output := make(map[string]map[string]Zim)
|
||||
comms := make(chan result)
|
||||
|
||||
for _, t := range getTypes() {
|
||||
page := getPage(t)
|
||||
links := getLinks(page)
|
||||
for _, lang := range getLanguages() {
|
||||
if file, err := getName(links, t, lang); err == nil {
|
||||
if _, ok := output[lang]; !ok {
|
||||
output[lang] = make(map[string]Zim)
|
||||
}
|
||||
output[lang][t] = Zim{ file, "" }
|
||||
go getHash(comms, file, t, lang)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for r := range comms {
|
||||
if entry, ok := output[r.language][r.category]; ok {
|
||||
entry.Hash = r.hash
|
||||
output[r.language][r.category] = entry
|
||||
}
|
||||
if outputIsValid(output) {
|
||||
close(comms)
|
||||
break
|
||||
}
|
||||
}
|
||||
ret, _ := json.Marshal(output)
|
||||
fmt.Println(string(ret))
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
writeShellApplication,
|
||||
nix-prefetch,
|
||||
go,
|
||||
...
|
||||
}:
|
||||
let
|
||||
goscript = ./updater.go;
|
||||
in
|
||||
writeShellApplication {
|
||||
name = "update-zims";
|
||||
runtimeInputs = [
|
||||
go
|
||||
nix-prefetch
|
||||
];
|
||||
text = ''
|
||||
go run ${goscript}
|
||||
'';
|
||||
}
|
||||
Reference in New Issue
Block a user