Merge branch 'update-nixos' into 'develop'

Add a script to update nixos version, and run it.

Closes #83

See merge request !165

docs/source/dev/README.rst

@@ -39,6 +39,24 @@ Inside that is bin/nixos-test-driver which gives you a kind of REPL for interact
 The kind of `Python in this testScript <https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/blob/78881a3/nixos/modules/tests/private-storage.nix#L180>`_ is what you can enter into this REPL.
 Consult the `official documentation on NixOS Tests <https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests>`_ for more information.
 
+Updatings Pins
+--------------
+
+Nixpkgs
+```````
+
+To update the version of NixOS we deploy with, run:
+
+.. code: shell
+
+   nix-shell --run 'update-nixpkgs'
+
+That will update ``nixpkgs-2015.json`` to the latest release on the nixos-21.05 channel.
+
+To update the channel, the script will need to be updated,
+along with the filenames that have the channel in them.
+
+
 Architecture overview
 ---------------------
 

nixpkgs-2105.json

-{ "name": "release2105"
-, "url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.804.5de44c15758/nixexprs.tar.xz"
-, "sha256": "002zvc16hyrbs0icx1qj255c9dqjpdxx4bhhfjndlj3kwn40by0m"
+{
+  "name": "release2105",
+  "url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3065.b3083bc6933/nixexprs.tar.xz",
+  "sha256": "186vni8rij8bhd6n5n9h55jf2x78v9zdy2gn9v4cpjhajp4pvzm0"
 }

shell.nix

 let
   release2105 = import ./nixpkgs-2105.nix { };
 in
-{ pkgs ? release2105 }:
+{ pkgs ? release2105, lib ? pkgs.lib, python ? pkgs.python3 }:
+let
+  tools = pkgs.callPackage ./tools {};
+in
 pkgs.mkShell {
   # When a path (such as `pkgs.path`) is interpolated into a string then nix
   # first adds that path to the store, and then interpolates the store path
@@ -15,7 +18,10 @@ pkgs.mkShell {
   shellHook = ''
     export NIX_PATH="nixpkgs=${builtins.toString pkgs.path}";
   '';
+  # Run the shellHook from tools
+  inputsFrom = [tools];
   buildInputs = [
+    tools
     pkgs.morph
     pkgs.jp
   ];

tools/default.nix

+{ pkgs, lib, makeWrapper, ... }:
+let
+  python = pkgs.python3;
+  # This is a python envionment that has the dependencies
+  # for the development python scripts we use, and the
+  # helper library.
+  python-env = python.buildEnv.override {
+    extraLibs = [ python.pkgs.httpx ];
+    # Add `.pth` file pointing at the directory containg our helper library.
+    # This will get added to `sys.path` by `site.py`.
+    # See https://docs.python.org/3/library/site.html
+    postBuild = ''
+      echo ${lib.escapeShellArg ./pylib} > $out/${lib.escapeShellArg python.sitePackages}/tools.pth
+    '';
+  };
+  python-commands = [
+    ./update-nixpkgs
+  ];
+in
+  # This derivation creates a package that wraps our tools to setup an environment
+  # with there dependencies available.
+pkgs.runCommand "ps_tools" {
+  nativeBuildInputs = [ makeWrapper ];
+  shellHook = ''
+    # Only display the help if we are running an interactive shell.
+    if [[ $- == *i* ]]; then
+      cat <<MOTD
+    Tools (pass --help for details):
+    ${lib.concatStringsSep "\n" (map (path:
+        "- ${baseNameOf path}"
+    ) python-commands)}
+    MOTD
+    fi
+  '';
+  } ''
+    mkdir -p $out/bin
+    ${lib.concatStringsSep "\n" (map (path:
+      let
+        baseName = baseNameOf path;
+        # We use toString so that we wrap the in-tree scripts, rather than copying
+        # them to the nix-store. This means that we don't need to run nix-shell again
+        # to pick up changes.
+        sourcePath = toString path;
+      in
+      # makeWrapper <executable> <wrapperfile> <args>
+      # See https://nixos.org/manual/nixpkgs/stable/#fun-makeWrapper
+      "makeWrapper ${python-env}/bin/python $out/bin/${baseName} --add-flags ${sourcePath}"
+      ) python-commands)}
+  ''

tools/pylib/README.rst

+This directory contains a python package of helper functions used by the scripts in ``tools/``.
+To get this on the python path, run ``nix-shell`` in the root of the repository.

tools/pylib/ps_tools/__init__.py

+"""
+Helpers for development and CI scripts.
+"""
+from __future__ import annotations
+
+import subprocess
+
+
+def get_url_hash(hash_type, name, url) -> dict[str, str]:
+    """
+    Get the nix hash of the given URL.
+
+    :returns: Dictionary of arguments suitable to pass to :nix:`pkgs.fetchzip`
+        or a function derived from it (such as :nix:`pkgs.fetchFromGitLab`)
+        to specify the hash.
+    """
+    output = subprocess.run(
+        [
+            "nix-prefetch-url",
+            "--type",
+            hash_type,
+            "--unpack",
+            "--name",
+            name,
+            url,
+        ],
+        capture_output=True,
+        check=True,
+        encoding="utf-8",
+    )
+
+    return {
+        "outputHashAlgo": hash_type,
+        "outputHash": output.stdout.strip(),
+    }

tools/update-nixpkgs

+#!/usr/bin/env python
+
+import argparse
+import json
+from pathlib import Path
+
+import httpx
+from ps_tools import get_url_hash
+
+# We pass this to builtins.fetchTarball which only supports sha256
+HASH_TYPE = "sha256"
+
+DEFAULT_CHANNEL = "nixos-21.05"
+CHANNEL_URL_TEMPLATE = "https://channels.nixos.org/{channel}/nixexprs.tar.xz"
+
+
+def get_nixos_channel_url(*, channel):
+    """
+    Get the URL for the current release of the given nixos channel.
+
+    `https://channels.nixos.org/<channel>` redirects to the path on
+    `https://releases.nixos.org` that corresponds to the current release
+    of that channel. This captures that redirect, so we can pin against
+    the release.
+    """
+    response = httpx.head(
+        CHANNEL_URL_TEMPLATE.format(channel=channel), allow_redirects=False
+    )
+    response.raise_for_status()
+    assert response.is_redirect
+    return str(response.next_request.url)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Update a pinned nixos repository.")
+    parser.add_argument(
+        "repo_file",
+        metavar="repo-file",
+        nargs="?",
+        default=Path(__file__).parent.with_name("nixpkgs-2105.json"),
+        type=Path,
+        help="JSON file with pinned configuration.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+    )
+    parser.set_defaults(channel=DEFAULT_CHANNEL)
+    args = parser.parse_args()
+
+    repo_file = args.repo_file
+    config = json.loads(repo_file.read_text())
+
+    config["url"] = get_nixos_channel_url(channel=args.channel)
+    hash_data = get_url_hash(HASH_TYPE, name=config["name"], url=config["url"])
+    config["sha256"] = hash_data["outputHash"]
+
+    output = json.dumps(config, indent=2)
+    if args.dry_run:
+        print(output)
+    else:
+        repo_file.write_text(output)
+
+
+if __name__ == "__main__":
+    main()