From bb79e5aedf0b1b4c52404bfe5897328a80bfbe41 Mon Sep 17 00:00:00 2001 From: Greg Hellings Date: Tue, 2 May 2023 21:33:02 -0500 Subject: [PATCH] Add multi-python post --- content/posts/multiple-version-testing.md | 244 ++++++++++++++++++++++ flake.lock | 6 +- 2 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 content/posts/multiple-version-testing.md diff --git a/content/posts/multiple-version-testing.md b/content/posts/multiple-version-testing.md new file mode 100644 index 0000000..ebd6a56 --- /dev/null +++ b/content/posts/multiple-version-testing.md @@ -0,0 +1,244 @@ +--- +title: "Multiple Version Testing" +date: 2023-05-02T11:07:07-05:00 +draft: false +--- +If you are interested in running multiple versions of the same development language using +Nix and Nix Flakes, [ride along](https://src.thehellings.com/greg/multi-python-demo) +while I setup a development environment that includes +Python 2.7, 3.5-3.12 along with necessary development libraries. I will not be covering +installing Nix on your system or the basics of the Nix package system or language. At the +link above, you can see and fetch the flake.nix file. Each commit in the repository +corresponds to a version of the file in this post. + +If you only want to see the full flake file and get to using it for yourself, check down +[at the bottom of the post](#full-array). + +**Caveat: This post assumes you have basic knowledge of Nix, its installation, and its +file syntax.** + +# Initial Flake Setup + +Create a new folder and create your `flake.nix` file with the basic inputs. Tell it to +install the Python 3 package and a shell to use. +```nix +{ + description = "A Python development environment"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + flake-utils, + nixpkgs + }: flake-utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages.${system}; + in { + devShell = pkgs.mkShell { + nativeBuildInputs = [ pkgs.bashInteractive ]; + buildInputs = [ + pkgs.python3 + ]; + }; + }); +} +``` +Enter the development environment with the command `nix develop`. This will put you into +a shell with the Nix packages defined in the `flake.nix` file. Of course, if you add +flake.nix and the generated flake.lock files to your repository, you can be sure that +everyone else who works on a project is able to use the exact same set of packages across +their systems. + +Personally, I use direnv tied into my shell to automatically activate my devshells, +but for the purposes of this exercise I am going to use the `nix develop` command to +build and test the environments. Setting up direnv and distributing Nix flakes is beyond +the scope of this post + +With the current version of Nix unstable, I get the following output: +```terminal +[greg@jude:~/tmp]$ python3 --version +Python 3.10.11 +``` + +# Add Python Packages + +When developing with Python, it is common to use a number of Python packages. Nixpkgs comes with +a huge number of them already packaged and it is easy to package more for it. For the purpose of +demonstration, I will only add one packge to my installation, but you can add any Python packages +that you need. To do this, we are going to call the `withPackages` function on the Python +package and give it a function that returns the packages we want added. That function will look +like this +```nix + packages = python-packages: with python-packages; [ + requests + ]; +``` +Similarly we invoke the function by changing +```diff +-pkgs.python3 ++(pkgs.python3.withPackages packages) +``` + +When that is added to the Flake file, the whole file looks like this +```nix +{ + description = "A Python development environment"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + flake-utils, + nixpkgs + }: flake-utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages.${system}; + packages = python-packages: with python-packages; [ + requests + ]; + in { + devShell = pkgs.mkShell { + nativeBuildInputs = [ pkgs.bashInteractive ]; + buildInputs = [ + (pkgs.python3.withPackages packages) + ]; + }; + }); +} +``` + +Now I can run `nix develop` again and get a shell where I can import the `requests` library +in my Python interpreter. + +# Add more versions + +Having one version of Python is good, but the point is to enable working with different versions. +To do this, we will create a Flake input for each version from Python 3.5 through 3.11 that we +have available. Creating a separate input, even when several of them will point to the same nixpkgs git +branch, is necessary because each version will leave support at a different time. We want to be +able to update the flake in the future and easily leave unsupported versions of Python on their +latest Nix branch while also not holding back supported versions from their future patch releases. +We also add a pin for Python 2.7, which is still in use in some legacy servers. + +So let's add a bunch of inputs like so: +```nix +nixpkgs311.url = "github:NixOS/nixpkgs/nixos-unstable"; +nixpkgs310.url = "github:NixOS/nixpkgs/nixos-22.11"; +nixpkgs39.url = "github:NixOS/nixpkgs/nixos-22.11"; +nixpkgs38.url = "github:NixOS/nixpkgs/nixos-22.11"; +nixpkgs37.url = "github:NixOS/nixpkgs/nixos-22.05"; +nixpkgs36.url = "github:NixOS/nixpkgs/nixos-21.05"; +nixpkgs35.url = "github:NixOS/nixpkgs/nixos-20.03"; +nixpkgs27.url = "github:NixOS/nixpkgs/nixos-20.09"; + +``` + +At the time of writing NixOS 22.11 is the latest release and Python 3.8-3.11 are all still in +support. Therefore, all of them are available in the 22.11 branch for the time being. At the same +time we do this, we add the new inputs to the output function, update the identification of +system support, and add the package to the available list. + +# Full Array {#full-array} + +Now our flake looks like this: +```nix +{ + description = "A Python development environment"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + nixpkgs311.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs310.url = "github:NixOS/nixpkgs/nixos-22.11"; + nixpkgs39.url = "github:NixOS/nixpkgs/nixos-22.11"; + nixpkgs38.url = "github:NixOS/nixpkgs/nixos-22.11"; + nixpkgs37.url = "github:NixOS/nixpkgs/nixos-22.05"; + nixpkgs36.url = "github:NixOS/nixpkgs/nixos-21.05"; + nixpkgs35.url = "github:NixOS/nixpkgs/nixos-20.03"; + nixpkgs27.url = "github:NixOS/nixpkgs/nixos-20.09"; + }; + + outputs = { + self, + flake-utils, + nixpkgs, + nixpkgs311, + nixpkgs310, + nixpkgs39, + nixpkgs38, + nixpkgs37, + nixpkgs36, + nixpkgs35, + nixpkgs27, + }: flake-utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages.${system}; + pkgs311 = nixpkgs311.legacyPackages.${system}; + pkgs310 = nixpkgs310.legacyPackages.${system}; + pkgs39 = nixpkgs39.legacyPackages.${system}; + pkgs38 = nixpkgs38.legacyPackages.${system}; + pkgs37 = nixpkgs37.legacyPackages.${system}; + pkgs36 = nixpkgs36.legacyPackages.${system}; + pkgs35 = nixpkgs35.legacyPackages.${system}; + pkgs27 = nixpkgs27.legacyPackages.${system}; + packages = python-packages: with python-packages; [ + requests + ]; + in { + devShell = pkgs.mkShell { + nativeBuildInputs = [ pkgs.bashInteractive ]; + buildInputs = [ + (pkgs.python3.withPackages packages) + (pkgs311.python311.withPackages packages) + (pkgs310.python310.withPackages packages) + (pkgs39.python39.withPackages packages) + (pkgs38.python38.withPackages packages) + (pkgs37.python37.withPackages packages) + (pkgs36.python36.withPackages packages) + (pkgs35.python35.withPackages packages) + (pkgs27.python27.withPackages packages) + ]; + }; + }); +} +``` + +We can test that they are all present by doing a `nix develop`. This build might take a long time the +first time you run it on your machine, because older packages are not available in the binary +cache for NixOS. Fortunately, all Nix packages include the entire history of how to build them from +source, so you CAN get it done. It just might take a while. Let's run a tiny Bash script to check +that we have all the Python versions and that they all have the Requests library available: + +```bash +for ver in $(seq 5 11); do + python3.${ver} -c 'import sys, requests; print(sys.version_info)' +done +``` + +Sample output: +```terminal +$ for ver in $(seq 5 11); do python3.${ver} -c 'import requests, sys; print(sys.version_info)'; done +sys.version_info(major=3, minor=5, micro=9, releaselevel='final', serial=0) +sys.version_info(major=3, minor=6, micro=14, releaselevel='final', serial=0) +sys.version_info(major=3, minor=7, micro=16, releaselevel='final', serial=0) +sys.version_info(major=3, minor=8, micro=16, releaselevel='final', serial=0) +sys.version_info(major=3, minor=9, micro=16, releaselevel='final', serial=0) +sys.version_info(major=3, minor=10, micro=11, releaselevel='final', serial=0) +sys.version_info(major=3, minor=11, micro=3, releaselevel='final', serial=0) +``` + +Likewise we can validate that Python 2.7 is available: +```terminal +$ python2.7 -c 'import requests, sys; print(sys.version_info)' +sys.version_info(major=2, minor=7, micro=18, releaselevel='final', serial=0) +``` + +# Python 3.0 to 3.4 + +These versions of Python were released before Nix was its current, modern self. There is not support +in those Nixpkgs releases for Flakes which makes adding them to the environment a pain. I tried testing +importing the tarball directly, but the actual builds of the old versions were broken. Versions 3.2-3.4 are in +the repository but they do not build without major changes which would be beyond the scope of this +post. Meanwhile Python 3.0 and 3.1 were released before Nix packages even existed and were never +packaged for the main repository. diff --git a/flake.lock b/flake.lock index 561ff7f..00ddd0f 100644 --- a/flake.lock +++ b/flake.lock @@ -17,11 +17,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1665296151, - "narHash": "sha256-uOB0oxqxN9K7XGF1hcnY+PQnlQJ+3bP2vCn/+Ru/bbc=", + "lastModified": 1682960002, + "narHash": "sha256-5Zjh4pT3lAjFGN1gVrjqj1LLJHKCAlGdLD8raU7oEMc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "14ccaaedd95a488dd7ae142757884d8e125b3363", + "rev": "8670e496ffd093b60e74e7fa53526aa5920d09eb", "type": "github" }, "original": {