From a5860e8076767ff010f2703187753937d3ea0b15 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 7 Mar 2023 17:17:24 -0600 Subject: [PATCH] Add icdm-root --- flake.nix | 1 + hosts/icdm-root/boot.nix | 11 ++++ hosts/icdm-root/default.nix | 24 +++++++++ hosts/icdm-root/filesystem.nix | 12 +++++ hosts/icdm-root/hardware-configuration.nix | 45 ++++++++++++++++ hosts/icdm-root/location.nix | 15 ++++++ hosts/icdm-root/networking.nix | 20 +++++++ hosts/icdm-root/wiki.nix | 32 +++++++++++ modules | 1 - modules-linux/default.nix | 1 + modules-linux/kiwix-serve.nix | 48 +++++++++++++++++ overlays/default.nix | 3 ++ overlays/enwiki-dump.nix | 55 +++++++++++++++++++ overlays/kiwix-tools.nix | 48 +++++++++++++++++ overlays/libkiwix.nix | 62 ++++++++++++++++++++++ 15 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 hosts/icdm-root/boot.nix create mode 100644 hosts/icdm-root/default.nix create mode 100644 hosts/icdm-root/filesystem.nix create mode 100644 hosts/icdm-root/hardware-configuration.nix create mode 100644 hosts/icdm-root/location.nix create mode 100644 hosts/icdm-root/networking.nix create mode 100644 hosts/icdm-root/wiki.nix delete mode 120000 modules create mode 100644 modules-linux/kiwix-serve.nix create mode 100644 overlays/enwiki-dump.nix create mode 100644 overlays/kiwix-tools.nix create mode 100644 overlays/libkiwix.nix diff --git a/flake.nix b/flake.nix index 4bce827..9eba4ca 100644 --- a/flake.nix +++ b/flake.nix @@ -69,6 +69,7 @@ nixosConfigurations = { "2maccabees" = machine { system = "aarch64-linux"; name = "2maccabees"; channel = nixunstable; }; jude = machine { name = "jude"; channel = nixunstable; }; + "icdm-root" = machine { name = "icdm-root"; }; "linode" = machine { name = "linode"; }; "lappy" = machine { name = "lappy"; }; "iso" = machine { name = "iso"; }; diff --git a/hosts/icdm-root/boot.nix b/hosts/icdm-root/boot.nix new file mode 100644 index 0000000..796db1b --- /dev/null +++ b/hosts/icdm-root/boot.nix @@ -0,0 +1,11 @@ +{ pkgs, config, ...}: + +{ + # Bootloader. + boot = { + loader.grub = { + enable = true; + device = "/dev/sda"; + }; + }; +} diff --git a/hosts/icdm-root/default.nix b/hosts/icdm-root/default.nix new file mode 100644 index 0000000..27aae2f --- /dev/null +++ b/hosts/icdm-root/default.nix @@ -0,0 +1,24 @@ +# Edit this configuration file to define what should be installed on +# your system. Help is available in the configuration.nix(5) man page +# and in the NixOS manual (accessible by running ‘nixos-help’). + +{ config, pkgs, agenix, ... }: + +{ + imports = [ # Include the results of the hardware scan. + ./hardware-configuration.nix + ./boot.nix + ./filesystem.nix + ./location.nix + ./networking.nix + ./wiki.nix + ]; + + # Define a user account. Don't forget to set a password with ‘passwd’. + users.users.greg = { + isNormalUser = true; + description = "Gregory Hellings"; + extraGroups = [ "networkmanager" "wheel" ]; + packages = with pkgs; []; + }; +} diff --git a/hosts/icdm-root/filesystem.nix b/hosts/icdm-root/filesystem.nix new file mode 100644 index 0000000..25b6e9a --- /dev/null +++ b/hosts/icdm-root/filesystem.nix @@ -0,0 +1,12 @@ +{ ... }: + +let +in { + fileSystems."serve" = { + #device = "10.42.1.4:/volume1/icdm-mysql/"; + #fsType = "nfs"; + device = "/dev/sdb1"; + fsType = "auto"; + mountPoint = "/srv"; + }; +} diff --git a/hosts/icdm-root/hardware-configuration.nix b/hosts/icdm-root/hardware-configuration.nix new file mode 100644 index 0000000..0919a73 --- /dev/null +++ b/hosts/icdm-root/hardware-configuration.nix @@ -0,0 +1,45 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = + [ (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ]; + boot.initrd.kernelModules = [ ]; + boot.kernelModules = [ "kvm-intel" ]; + boot.extraModulePackages = [ ]; + + fileSystems."/" = + { device = "/dev/disk/by-uuid/dab0d455-e25e-4445-8fa4-5320047d7e7b"; + fsType = "btrfs"; + options = [ "subvol=@" ]; + }; + + fileSystems."/boot" = + { device = "/dev/disk/by-uuid/5aedbb07-5761-423b-909d-2560405eae32"; + fsType = "ext4"; + }; + + fileSystems."/var" = + { device = "/dev/disk/by-uuid/57968536-c29d-417d-997e-85223d1d1f65"; + fsType = "btrfs"; + }; + + swapDevices = + [ { device = "/dev/disk/by-uuid/09691dce-375a-43c6-8d40-4498d20a6d9a"; } + ]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces..useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.eno1.useDHCP = lib.mkDefault true; + # networking.interfaces.wlp2s0.useDHCP = lib.mkDefault true; + + hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/hosts/icdm-root/location.nix b/hosts/icdm-root/location.nix new file mode 100644 index 0000000..d2d0d72 --- /dev/null +++ b/hosts/icdm-root/location.nix @@ -0,0 +1,15 @@ +{...}: + +{ + # Set your time zone. + time.timeZone = "America/Chicago"; + + # Select internationalisation properties. + i18n.defaultLocale = "en_US.UTF-8"; + + # Configure keymap in X11 + services.xserver = { + layout = "us"; + xkbVariant = ""; + }; +} diff --git a/hosts/icdm-root/networking.nix b/hosts/icdm-root/networking.nix new file mode 100644 index 0000000..8f11a2d --- /dev/null +++ b/hosts/icdm-root/networking.nix @@ -0,0 +1,20 @@ +{...}: + +{ + # If we have to do proxying in Bayonnais, we can start to work on that here + # networking.proxy.noProxy = "127.0.0.1,localhost,internal.domain"; + networking = { + hostName = "icdm-root"; + useDHCP = false; + defaultGateway = "10.42.1.1"; + nameservers = [ "100.100.100.100" "10.42.1.2" ]; + enableIPv6 = false; + + interfaces = { + eno1.ipv4.addresses = [ { + address = "10.42.101.1"; + prefixLength = 16; + } ]; + }; + }; +} diff --git a/hosts/icdm-root/wiki.nix b/hosts/icdm-root/wiki.nix new file mode 100644 index 0000000..46ab920 --- /dev/null +++ b/hosts/icdm-root/wiki.nix @@ -0,0 +1,32 @@ +{ pkgs, ... }: +let + wikiPasswordFile = "/var/mediawiki/passwordFile"; + wikiHost = "wiki.icdm.lan"; + kiwixport = 8080; + dependents = with pkgs; [ + enwiki-dump + transmission + ]; +in +{ + services.kiwix-serve = { + enable = true; + port = kiwixport; + path = "/srv/zims/*.zim"; + }; + + services.transmission = { + enable = true; + settings = { + download-dir = "/srv"; + incomplete-dir = "/srv/incomplete"; + rpc-bind-address = "0.0.0.0"; + rpc-whitelist = "10.42.*,127.*,localhost"; + }; + }; + + greg.proxies."${wikiHost}".target = "http://localhost:${toString kiwixport}"; + networking.firewall.allowedTCPPorts = [ 80 ]; + + environment.systemPackages = dependents; +} diff --git a/modules b/modules deleted file mode 120000 index 89a6caa..0000000 --- a/modules +++ /dev/null @@ -1 +0,0 @@ -modules-all \ No newline at end of file diff --git a/modules-linux/default.nix b/modules-linux/default.nix index 9b25e49..1a9fd2d 100644 --- a/modules-linux/default.nix +++ b/modules-linux/default.nix @@ -6,6 +6,7 @@ ./backup.nix ./gnome.nix ./home.nix + ./kiwix-serve.nix ./linode.nix ./linux.nix ./proxy.nix diff --git a/modules-linux/kiwix-serve.nix b/modules-linux/kiwix-serve.nix new file mode 100644 index 0000000..ef714a2 --- /dev/null +++ b/modules-linux/kiwix-serve.nix @@ -0,0 +1,48 @@ +{ config, pkgs, lib, ... }: + +let + cfg = config.services.kiwix-serve; +in with lib; { + options.services.kiwix-serve = { + enable = mkEnableOption "Enable the Kiwix web server"; + + port = mkOption { + type = types.int; + default = 8888; + description = "Port to serve the Kiwix HTTP service on"; + }; + + path = mkOption { + type = types.str; + default = "/var/lib/kiwix-serve/"; + description = "Path to Zim file(s) to serve"; + }; + + proxy = mkOption { + type = types.str; + default = ""; + description = ''Upstream proxy, if any, to configure with kiwix. Specify + host and port. E.g. "localhost:8080" + ''; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ + pkgs.kiwix-tools + ]; + + systemd.services.kiwix-serve = { + enable = true; + after = [ "network.service" ]; + description = "Runs the kiwix-serve binary as a sysmted service"; + restartTriggers = [ pkgs.kiwix-tools ]; + wantedBy = [ "multi-user.target" ]; + script = "${pkgs.kiwix-tools}/bin/kiwix-serve --port ${toString cfg.port} ${cfg.path}"; + environment = { + UPSTREAM_HOST = mkIf (cfg.proxy != "") cfg.proxy; + UPSTREAM_WIKI = mkIf (cfg.proxy != "") cfg.proxy; + }; + }; + }; +} diff --git a/overlays/default.nix b/overlays/default.nix index 21067be..22fdcfb 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -58,9 +58,12 @@ in rec { xonsh-direnv = cp ./xonsh-direnv.nix {}; }; + enwiki-dump = super.callPackage ./enwiki-dump.nix {}; hms = super.callPackage ./hms.nix { pkgs = self.pkgs; }; + kiwix-tools = super.callPackage ./kiwix-tools.nix {}; + libkiwix = super.callPackage ./libkiwix.nix {}; xonsh = super.xonsh.overridePythonAttrs (old: rec{ python3 = self.gregpy; diff --git a/overlays/enwiki-dump.nix b/overlays/enwiki-dump.nix new file mode 100644 index 0000000..80f9aaf --- /dev/null +++ b/overlays/enwiki-dump.nix @@ -0,0 +1,55 @@ +{ pkgs, ... }: + +let + version = "2022.07.20"; + date = builtins.replaceStrings ["."] [""] version; + +in pkgs.writeShellScriptBin "wiki-data" '' +set -eo pipefail + +function process_file { + file="''${lang}''${1}" + destfile=/var/tmp/''${file} + curl -C - -o "''${destfile}" "https://dumps.wikimedia.your.org/''${lang}wiki/${date}/''${file}" + bzcat "''${destfile}" | ${pkgs.php}/bin/php ${pkgs.mediawiki}/share/mediawiki/maintenance/importDump.php --report 500 | tee "/var/log/mediawiki-''${lang}-import" + rm "''${destfile}" +} + +function zim_fetch { + dest="/srv/zims" + mkdir -p "''${dest}" + file="''${1}" + echo "Downloading ''${file}" + transmission-remote -w "''${dest}" -a https://download.kiwix.org/zim/''${file}.zim.torrent +} + +#process_file "wiki-${date}-pages-articles-multistream.xml.bz2" +#process_file "wiki-${date}-pages-meta-current.xml.bz2" +#process_file "wiki-${date}-pages-articles.xml.bz2" + +zim_fetch wikipedia_en_all_maxi +zim_fetch wikipedia_fr_all_maxi +zim_fetch wikipedia_ht_all_maxi + +zim_fetch wiktionary_en_all_maxi +zim_fetch wiktionary_fr_all_maxi + +zim_fetch wikiversity_en_all_maxi +zim_fetch wikiversity_fr_all_maxi + +zim_fetch wikibooks_en_all_maxi +zim_fetch wikibooks_fr_all_maxi + +zim_fetch wikisource_en_all_maxi +zim_fetch wikisource_fr_all_maxi + +zim_fetch ted_en_science +zim_fetch ted_en_technology + +zim_fetch phet_en +zim_fetch phet_fr +zim_fetch phet_ht + +zim_fetch gutenberg_en_all +zim_fetch gutenberg_fr_all +'' diff --git a/overlays/kiwix-tools.nix b/overlays/kiwix-tools.nix new file mode 100644 index 0000000..46f3934 --- /dev/null +++ b/overlays/kiwix-tools.nix @@ -0,0 +1,48 @@ +{ lib, pkgs, stdenv + , cmake + , meson + , ninja + , pkg-config + + , curl + , libkiwix + , libmicrohttpd + , pugixml + , zimlib + , ...}: + +stdenv.mkDerivation rec { + pname = "kiwix-tools"; + version = "3.1.2"; + + src = pkgs.fetchFromGitHub { + owner = "kiwix"; + repo = pname; + rev = version; + # For 3.3.0 + #sha256 = "sha256-jeIyHTbgGrYknLaTNNSFVFo9zXyyGZU6dIVv49bEhBw="; + sha256 = "sha256-i3wr0VZCeH3aRCgphIyn6U91Ja+qVT4NVAzJSmniu8o="; + }; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + curl + libkiwix + libmicrohttpd + pugixml + zimlib + ]; + + meta = with lib; { + description = "An offline read for Web content tools (HTTP server, search, etc)"; + homepage = "https://kiwix.org"; + license = licenses.gpl3Plus; + platforms = platforms.all; + maintainers = [ "Greg Hellings " ]; + }; +} diff --git a/overlays/libkiwix.nix b/overlays/libkiwix.nix new file mode 100644 index 0000000..ce7f6a7 --- /dev/null +++ b/overlays/libkiwix.nix @@ -0,0 +1,62 @@ +{ lib, pkgs, stdenv + , curl + , icu + , pugixml + , mustache-hpp + , libmicrohttpd + , zimlib + , zlib + + , bash + , gcc + , gtest + , meson + , ninja + , pkg-config + , python3 + , ...}: + +stdenv.mkDerivation rec { + pname = "libkiwix"; + version = "9.4.1"; + + src = pkgs.fetchFromGitHub { + owner = "kiwix"; + repo = pname; + rev = version; + sha256 = "sha256-vE8bJrNx0oojTgcDhlr+SyfzdpigCZ0pQ2cPYaiZlgw="; + }; + + nativeBuildInputs = [ + bash + gcc + gtest + meson + ninja + pkg-config + python3 + ]; + + buildInputs = [ + curl + icu + libmicrohttpd + mustache-hpp + pugixml + zimlib + zlib + ]; + + mesonFlags = [ "--debug" ]; + + # Built-in python script uses /usr/bin/env + patchPhase = "patchShebangs --build scripts/kiwix-compile-resources"; + + meta = with lib; { + description = "Kiwix file library for use by other applications"; + homepage = "https://kiwix.org"; + license = licenses.gpl3Plus; + platforms = platforms.all; + maintainers = [ "Greg Hellings " ]; + }; +}