morph/lib/issuer-aws.nix

@@ -6,22 +6,48 @@
   boot.loader.grub.device = lib.mkForce "/dev/nvme0n1";
 
   ec2.hvm = true;
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
   swapDevices = [ {
     device = "/var/swapfile";
-    size = 4096; # megabytes
+    size = 1024; # megabytes
     randomEncryption = true;
   } ];
 
+  # If we don't manually and explicitly early-load the loop module, crypt-swap
+  # setup fails with the not very helpful message: "loop device with autoclear
+  # flag is required"
+  # See https://unix.stackexchange.com/a/554500/81275
+  boot.kernelModules = [ "loop" ];
+
+  # NixOS likes to fill up boot partitions with (by default) 100 old kernels.
+  # Keep a (for us) more reasonable number around.
+  boot.loader.grub.configurationLimit = 8;
+
   # Break the tie between AWS and morph for the hostname by forcing the
   # morph-supplied name.  See also
   # <https://github.com/DBCDK/morph/issues/146>.
   networking.hostName = name;
 
+  # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+  # beyond control of this particular part of the system) for the
+  # PaymentServer voucher database.  This makes it easier to manage for
+  # tasks like backup/recovery and encryption.
+  services.private-storage-issuer.databaseFileSystem = {
+    label = "zkapissuer-data";
+  };
+
   # Clean up packages after a while
   nix.gc = {
     automatic = true;
     dates = "weekly";
     options = "--delete-older-than 30d";
   };
+
+  # Turn on automatic optimization of nix store
+  # https://nixos.wiki/wiki/Storage_optimization
+  nix.settings.auto-optimise-store = true;
+
+  # Most of the time, we have ample free & usable memory, but when upgrading
+  # software, we sometimes run out because of Nix.  This is supposed to help:
+  zramSwap.enable = true;
 }

morph/lib/issuer-monitoring-ovh.nix

+{ modulesPath, name, lib, ... }: {
+
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+  boot.loader.grub.device = "/dev/sda";
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" ];
+  boot.initrd.kernelModules = [ "nvme" ];
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
+
+  fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
+
+  swapDevices = [ {
+    device = "/var/swapfile";
+    size = 4096; # megabytes
+    randomEncryption = true;
+  } ];
+
+  # Break the tie between AWS and morph for the hostname by forcing the
+  # morph-supplied name.  See also
+  # <https://github.com/DBCDK/morph/issues/146>.
+  networking.hostName = name;
+
+  # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+  # beyond control of this particular part of the system) for the
+  # PaymentServer voucher database.  This makes it easier to manage for
+  # tasks like backup/recovery and encryption.
+  services.private-storage-issuer.databaseFileSystem = {
+    label = "zkapissuer-data";
+  };
+
+  # Clean up packages after a while
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 30d";
+  };
+
+  # Turn on automatic optimization of nix store
+  # https://nixos.wiki/wiki/Storage_optimization
+  nix.settings.auto-optimise-store = true;
+}

morph/lib/issuer-payments-ovh.nix

+{ modulesPath, name, lib, ... }: {
+
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+  boot.loader.grub.device = "/dev/sda";
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
+
+  fileSystems."/" = { device = "/dev/sda3"; fsType = "ext4"; };
+  
+  swapDevices = [ {
+    device = "/dev/sda2";
+    randomEncryption = true;
+  } ];
+
+  # Break the tie between AWS and morph for the hostname by forcing the
+  # morph-supplied name.  See also
+  # <https://github.com/DBCDK/morph/issues/146>.
+  networking.hostName = name;
+
+  # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+  # beyond control of this particular part of the system) for the
+  # PaymentServer voucher database.  This makes it easier to manage for
+  # tasks like backup/recovery and encryption.
+  services.private-storage-issuer.databaseFileSystem = {
+    label = "zkapissuer-data";
+  };
+
+  # Clean up packages after a while
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 30d";
+  };
+
+  # Turn on automatic optimization of nix store
+  # https://nixos.wiki/wiki/Storage_optimization
+  nix.settings.auto-optimise-store = true;
+}

morph/lib/issuer.nix

-# This, along with `customize-issuer.nix, contains all of the NixOS system
-# configuration necessary to specify an "issuer"-type system.  Originally, this
-# file has all the static configuration, and `customize-issuer.nix` was a function
-# that filled in the holes. We are in the process of merging the modules, using settings
-# instead of function arguments.
-# See https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/80
-{ config, ...}:
+# This contains all of the NixOS system configuration necessary to specify an
+# "issuer"-type system.
+{ lib, config, ...}:
 let
-  inherit (config.grid) publicKeyPath privateKeyPath;
+  inherit (config.grid) privateKeyPath;
+  inherit (config.grid.issuer) issuerDomains allowedChargeOrigins tokensPerVoucher;
 in {
+  imports = [
+    ./monitoringvpn-client.nix
+  ];
+
+  options.grid.issuer = {
+    issuerDomains = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of strings giving the domain names that point at this issuer
+        system.  These will all be included in Let's Encrypt certificate.
+      '';
+    };
+
+    allowedChargeOrigins = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of strings giving CORS Origins will the issuer will be configured
+        to allow.
+      '';
+    };
+
+    tokensPerVoucher = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.int;
+      example = 50000;
+      description = ''
+        If not null, a value to pass to PaymentServer for
+        ``--tokens-per-voucher``.
+      '';
+    };
+  };
+
+  config = {
     deployment = {
       secrets = {
         "ristretto-signing-key" = {
@@ -26,36 +56,32 @@ in {
           permissions = "0400";
           action = ["sudo" "systemctl" "restart" "zkapissuer.service"];
         };
-
-      "monitoringvpn-secret-key" = {
-        destination = "/run/keys/monitoringvpn/client.key";
-        owner.user = "root";
-        owner.group = "root";
-        permissions = "0400";
-        action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
-      };
-      "monitoringvpn-preshared-key" = {
-        destination = "/run/keys/monitoringvpn/preshared.key";
-        owner.user = "root";
-        owner.group = "root";
+        "stripe-webhook-secret-key" = {
+          destination = "/run/keys/stripe.webhook-secret-key";
+          source = "${privateKeyPath}/stripe.webhook-secret";
+          owner.user = "zkapissuer";
+          owner.group = "zkapissuer";
           permissions = "0400";
-        action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
+          action = ["sudo" "systemctl" "restart" "zkapissuer.service"];
         };
       };
     };
-
-  imports = [
-    ../../nixos/modules/issuer.nix
-    ../../nixos/modules/monitoring/vpn/client.nix
-    ../../nixos/modules/monitoring/exporters/node.nix
-  ];
-
     services.private-storage-issuer = {
       enable = true;
       tls = true;
       ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination;
       stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
+      stripeWebhookSecretKeyPath = config.deployment.secrets.stripe-webhook-secret-key.destination;
       database = "SQLite3";
-    databasePath = "/var/lib/zkapissuer/vouchers.sqlite3";
+      databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
+      inherit (config.grid) letsEncryptAdminEmail;
+      inherit allowedChargeOrigins;
+      domains = issuerDomains;
+      inherit tokensPerVoucher;
+    };
+
+    services.private-storage.monitoring.exporters.node.enable = true;
+
+    system.stateVersion = "19.03";
   };
 }

morph/lib/monitoring.nix

-# Similar to ``issuer.nix`` but for a "monitoring"-type system.  Holes are
-# filled by ``customize-monitoring.nix``.
+# This contains all of the NixOS system configuration necessary to specify an
+# "monitoring"-type system.
+{ lib, config, nodes, ...}:
+let
+  cfg = config.grid.monitoring;
+  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnIPv4 letsEncryptAdminEmail;
+
+  # This collects information about monitored hosts from their configuration for use below.
+  monitoringHosts = lib.mapAttrsToList (name: node: rec {
+    inherit name;
+    vpnIPv4 = node.config.grid.monitoringvpnIPv4;
+    vpnHostName = "${name}.monitoringvpn";
+    hostNames = [name vpnHostName];
+  }) nodes;
+
+  # A set mapping VPN IP addresses as strings to lists of hostnames as
+  # strings.  The system's ``/etc/hosts`` will be populated with this
+  # information.  Apart from helping with normal forward resolution, this
+  # *also* gives us reverse resolution from the VPN IPs to hostnames which
+  # allows Grafana to show us hostnames instead of VPN IP addresses.
+  hostsMap = lib.listToAttrs (map (node: lib.nameValuePair node.vpnIPv4 node.hostNames) monitoringHosts);
+  # A list of VPN IP addresses as strings indicating which clients will be
+  # allowed onto the VPN.
+  vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts);
+  # A list of VPN clients (IP addresses or hostnames) as strings indicating
+  # which nodes to scrape "nodeExporter" metrics from.
+  nodeExporterTargets = map (node: node.vpnHostName) monitoringHosts;
+in {
+  imports = [
+    ../../nixos/modules/monitoring/vpn/server.nix
+    ../../nixos/modules/monitoring/server/grafana.nix
+    ../../nixos/modules/monitoring/server/prometheus.nix
+    ../../nixos/modules/monitoring/server/loki.nix
+    ../../nixos/modules/monitoring/exporters/blackbox.nix
+  ];
+
+  options.grid.monitoring = {
+    paymentExporterTargets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of VPN clients (IP addresses or hostnames) as strings indicating
+        which nodes to scrape PaymentServer metrics from.
+      '';
+    };
+
+    blackboxExporterHttpsTargets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of HTTPS servers (URLs, IP addresses or hostnames) as strings indicating
+        which nodes the BlackboxExporter should scrape HTTP and TLS metrics from.
+      '';
+    };
+
+    monitoringDomains = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of strings giving the domain names that point at this monitoring
+        system.  These will all be included in Let's Encrypt certificate.
+      '';
+    };
+
+
+    googleOAuthClientID = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = ''
+        A string containing the GSuite OAuth2 ClientID to use to authenticate
+        logins to Grafana.
+      '';
+    };
+
+    enableSlackAlert = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to enable alerting via Slack.
+        When true requires a grafana-slack-url file (see private-keys/README.rst).
+      '';
+    };
+
+    enableZulipAlert = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to enable alerting via Zulip.
+        When true requires a grafana-zulip-url file (see private-keys/README.rst).
+      '';
+    };
+  };
+
+  config = {
+    assertions = [
+      {
+        assertion = let
+          vpnIPs = (map (node: node.vpnIPv4) monitoringHosts);
+        in vpnIPs == lib.unique vpnIPs;
+        message = ''
+          Duplicate grid.monitoringvpnIPv4 values specified for different nodes.
+        '';
+      }
+    ];
+
+    deployment.secrets = lib.mkMerge [
       {
-  deployment = {
-    secrets = {
         "monitoringvpn-private-key" = {
         destination = "/run/keys/monitoringvpn/server.key";
+        source = "${privateKeyPath}/monitoringvpn/server.key";
         owner.user = "root";
         owner.group = "root";
         permissions = "0400";
@@ -12,21 +112,74 @@
         };
         "monitoringvpn-preshared-key" = {
         destination = "/run/keys/monitoringvpn/preshared.key";
+        source = "${privateKeyPath}/monitoringvpn/preshared.key";
         owner.user = "root";
         owner.group = "root";
         permissions = "0400";
         action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
         };
+        "grafana-admin-password" = {
+          source = "${privateKeyPath}/grafana-admin.password";
+          destination = "/run/keys/grafana-admin.password";
+          owner.user = config.systemd.services.grafana.serviceConfig.User;
+          owner.group = config.users.users.grafana.group;
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "grafana.service"];
         };
+      }
+      (lib.mkIf (cfg.googleOAuthClientID != "") {
+        "grafana-google-sso-secret" = {
+          source = "${privateKeyPath}/grafana-google-sso.secret";
+          destination = "/run/keys/grafana-google-sso.secret";
+          owner.user = config.systemd.services.grafana.serviceConfig.User;
+          owner.group = config.users.users.grafana.group;
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "grafana.service"];
         };
-
-  imports = [
-    ../../nixos/modules/monitoring/vpn/server.nix
-    ../../nixos/modules/monitoring/server/grafana.nix
-    ../../nixos/modules/monitoring/server/prometheus.nix
-    ../../nixos/modules/monitoring/exporters/node.nix
-    ../../nixos/modules/monitoring/exporters/blackbox.nix
-    # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
-    # ../../nixos/modules/monitoring/server/loki.nix
+      })
+      (lib.mkIf cfg.enableSlackAlert {
+        "grafana-slack-url" = {
+          source = "${privateKeyPath}/grafana-slack-url";
+          destination = "/run/keys/grafana-slack-url";
+          owner.user = config.systemd.services.grafana.serviceConfig.User;
+          owner.group = config.users.users.grafana.group;
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "grafana.service"];
+        };
+      })
+      (lib.mkIf cfg.enableZulipAlert {
+        "grafana-zulip-url" = {
+          source = "${privateKeyPath}/grafana-zulip-url";
+          destination = "/run/keys/grafana-zulip-url";
+          owner.user = config.systemd.services.grafana.serviceConfig.User;
+          owner.group = config.users.users.grafana.group;
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "grafana.service"];
+        };
+      })
     ];
+
+    networking.hosts = hostsMap;
+
+    services.private-storage.monitoring.vpn.server = {
+      enable = true;
+      ip = monitoringvpnIPv4;
+      inherit vpnClientIPs;
+      pubKeysPath = "${publicKeyPath}/monitoringvpn";
+    };
+
+    services.private-storage.monitoring.prometheus = {
+      inherit nodeExporterTargets;
+      inherit (cfg) paymentExporterTargets blackboxExporterHttpsTargets;
+      nginxExporterTargets = [];
+    };
+
+    services.private-storage.monitoring.grafana = {
+      inherit (cfg) googleOAuthClientID enableSlackAlert enableZulipAlert;
+      inherit letsEncryptAdminEmail;
+      domains = cfg.monitoringDomains;
+    };
+
+    services.private-storage.monitoring.exporters.node.enable = true;
+  };
 }

morph/lib/monitoringvpn-client.nix

+{ lib, config, ...}:
+let
+  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4;
+in {
+  config = {
+    deployment = {
+      secrets = {
+        "monitoringvpn-secret-key" = {
+          destination = "/run/keys/monitoringvpn/client.key";
+          source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
+          owner.user = "root";
+          owner.group = "root";
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
+        };
+        "monitoringvpn-preshared-key" = {
+          destination = "/run/keys/monitoringvpn/preshared.key";
+          source = "${privateKeyPath}/monitoringvpn/preshared.key";
+          owner.user = "root";
+          owner.group = "root";
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
+        };
+      };
+    };
+
+    services.private-storage.monitoring.vpn.client = {
+      enable = true;
+      ip = monitoringvpnIPv4;
+      endpoint = monitoringvpnEndpoint;
+      endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
+    };
+  };
+}

morph/lib/storage.nix

-# Similar to ``issuer.nix`` but for a "storage"-type system.  Holes are filled
-# by ``customize-storage.nix``.
-{ config, ...} :
+# This contains all of the NixOS system configuration necessary to specify an
+# "storage"-type system.
+{ lib, config, ...} :
 let
-  inherit (config.grid) publicKeyPath privateKeyPath;
+  inherit (config.grid) privateKeyPath;
 in {
+  # Any extra NixOS modules to load on this server.
+  imports = [
+    ./monitoringvpn-client.nix
+    ./borgbackup.nix
+  ];
+
+  options.grid.storage = {
+    passValue = lib.mkOption {
+      type = lib.types.int;
+      description = ''
+        An integer giving the value of a single pass in byte×months.
+      '';
+    };
+
+    publicStoragePort = lib.mkOption {
+      type = lib.types.port;
+      description = ''
+        An integer giving the port number to include in Tahoe storage service
+        advertisements and on which to listen for storage connections.
+      '';
+    };
+  };
+
+  config = {
     deployment = {
       secrets = {
         "ristretto-signing-key" = {
@@ -17,37 +41,13 @@ in {
           # extract it from the tahoe-lafs nixos module somehow?
           action = ["sudo" "systemctl" "restart" "tahoe.storage.service"];
         };
-      "monitoringvpn-secret-key" = {
-        destination = "/run/keys/monitoringvpn/client.key";
-        owner.user = "root";
-        owner.group = "root";
-        permissions = "0400";
-        action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
-      };
-      "monitoringvpn-preshared-key" = {
-        destination = "/run/keys/monitoringvpn/preshared.key";
-        owner.user = "root";
-        owner.group = "root";
-        permissions = "0400";
-        action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
-      };
       };
     };
 
-  # Any extra NixOS modules to load on this server.
-  imports = [
-    # Bring in our module for configuring the Tahoe-LAFS service and other
-    # Private Storage-specific things.
-    ../../nixos/modules/private-storage.nix
-    # Connect to the monitoringvpn.
-    ../../nixos/modules/monitoring/vpn/client.nix
-    # Expose base system metrics over the monitoringvpn.
-    ../../nixos/modules/monitoring/exporters/node.nix
-    # Collect Tahoe OpenMetrics statistics.
-    ../../nixos/modules/monitoring/exporters/tahoe.nix
-  ];
+    services.private-storage.monitoring.exporters.node.enable = true;
+    services.private-storage.monitoring.exporters.tahoe.enable = true;
 
-  services.private-storage.monitoring.tahoe.enable = true;
+    services.private-storage.borgbackup.enable = lib.mkDefault true;
 
     # Turn on the Private Storage (Tahoe-LAFS) service.
     services.private-storage = {
@@ -55,5 +55,7 @@ in {
       enable = true;
       # Give it the Ristretto signing key to support authorization.
       ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination;
+      inherit (config.grid.storage) passValue publicStoragePort;
+    };
   };
 }

nixos/lib/default.nix

+{ callPackage }:
+{
+  /* A library of tools useful for writing tests with Nix.
+  */
+  testing = callPackage ./testing.nix { };
+}

nixos/lib/ip-util.nix

+# Thank you: https://gist.github.com/petabyteboy/558ffddb9aeb24e1eab2d5d6d021b5d7
+
+with import <nixpkgs/lib>;
+
+rec {
+  # FIXME: add case for negative numbers
+  pow = base: exponent: if exponent == 0 then 1 else fold (
+    x: y: y * base
+  ) base (
+    range 2 exponent
+  );
+
+  fromHexString = hex: foldl (
+    x: y: 16 * x + (
+      (
+        listToAttrs (
+          map (
+            x: nameValuePair (
+              toString x
+            ) x
+          ) (
+            range 0 9
+          )
+        ) // {
+          "a" = 10;
+          "b" = 11;
+          "c" = 12;
+          "d" = 13;
+          "e" = 14;
+          "f" = 15;
+        }
+      ).${y}
+    )
+  ) 0 (
+    stringToCharacters (
+      removePrefix "0x" (
+        hex
+      )
+    )
+  );
+
+  ipv4 = rec {
+
+    decode = address: foldl (
+      x: y: 256 * x + y
+    ) 0 (
+      map toInt (
+        splitString "." address
+      )
+    );
+
+    encode = num: concatStringsSep "." (
+      map (
+        x: toString (mod (num / x) 256)
+      ) (
+        reverseList (
+          genList (
+            x: pow 2 (x * 8)
+          ) 4
+        )
+      )
+    );
+
+    netmask = prefixLength: (
+      foldl (
+        x: y: 2 * x + 1
+      ) 0 (
+        range 1 prefixLength
+      )
+    ) * (
+      pow 2 (
+        32 - prefixLength
+      )
+    );
+
+    reverseZone = net: (
+      concatStringsSep "." (
+        reverseList (
+          splitString "." net
+        )
+      )
+    ) + ".in-addr.arpa";
+
+    eachAddress = net: prefixLength: genList (
+      x: decode (
+        x + (
+          decode net
+        )
+      )
+    ) (
+      pow 2 (
+        32 - prefixLength
+      )
+    );
+
+    networkOf = address: prefixLength: encode (
+      bitAnd (
+        decode address
+      ) (
+        netmask prefixLength
+      )
+    );
+
+    isInNetwork = net: address: networkOf address == net;
+
+    /* nixos-specific stuff */
+
+    findOwnAddress = config: net: head (
+      filter (
+        isInNetwork net
+      ) (
+        configuredAddresses config
+      )
+    );
+
+    configuredAddresses = config: concatLists (
+      mapAttrsToList (
+        name: iface: iface.ipv4.addresses
+      ) config.networking.interfaces
+    );
+
+  };
+
+  ipv6 = rec {
+
+    expand = address: (
+      replaceStrings ["::"] [(
+        concatStringsSep "0" (
+          genList (x: ":") (
+            9 - (count (x: x == ":") (stringToCharacters address))
+          )
+        )
+      )] address
+    ) + (
+      if hasSuffix "::" address then
+        "0"
+      else
+        ""
+    );
+
+    decode = address: map fromHexString (
+      splitString ":" (
+        expand address
+      )
+    );
+
+    encode = address: toLower (
+      concatStringsSep ":" (
+        map toHexString address
+      )
+    );
+
+    netmask = prefixLength: map (
+      x: if prefixLength > x + 16 then
+        (pow 2 16) - 1
+      else if prefixLength < x then
+        0
+      else
+        (
+          foldl (
+            x: y: 2 * x + 1
+          ) 0 (
+            range 1 (prefixLength - x)
+          )
+        ) * (
+          pow 2 (
+            16 - (prefixLength - x)
+          )
+        )
+    ) (
+      genList (
+        x: x * 16
+      ) 8
+    );
+
+    reverseZone = net: (
+      concatStringsSep "." (
+        concatLists (
+          reverseList (
+            map (
+              x: stringToCharacters (fixedWidthString 4 "0" x)
+            ) (
+              splitString ":" (
+                expand net
+              )
+            )
+          )
+        )
+      )
+    ) + ".ip6.arpa";
+
+    networkOf = address: prefixLength: encode (
+      zipListsWith bitAnd (
+        decode address
+      ) (
+        netmask prefixLength
+      )
+    );
+
+    isInNetwork = net: address: networkOf address == (expand net);
+
+    /* nixos-specific stuff */
+
+    findOwnAddress = config: net: head (
+      filter (
+        isInNetwork net
+      ) (
+        configuredAddresses config
+      )
+    );
+
+    configuredAddresses = config: concatLists (
+      mapAttrsToList (
+        name: iface: iface.ipv6.addresses
+      ) config.networking.interfaces
+    );
+
+  };
+}

nixos/lib/testing.nix

+{ ...}:
+{
+  /* Returns a string that runs tests from the Python code at the given path.
+
+     The Python code is loaded using *execfile* and the *test* global it
+     defines is called with the given keyword arguments.
+
+     Type: makeTestScript :: Path -> AttrSet -> String
+
+     Example:
+       testScript = (makeTestScript ./test_foo.py { x = "y"; });
+  */
+  makeTestScript = { testpath, kwargs ? {} }:
+    ''
+    # The driver runs pyflakes on this script before letting it
+    # run... Convince pyflakes that there is a `test` name.
+    def test():
+        pass
+    with open("${testpath}") as testfile:
+        exec(testfile.read(), globals())
+    # For simple types, JSON is compatible with Python syntax!
+    test(**${builtins.toJSON kwargs})
+    '';
+}

nixos/modules/100tb.nix

@@ -68,6 +68,7 @@ let
     { type = lib.types.str;
       example = "wwn-0x5000c500936410b9";
       description = "The ID of the disk on which to install grub.";
+      default = "nodev";
     };
   };
 in {
@@ -102,10 +103,11 @@ in {
   # harder to deploy in the bootstrap environment.
   config =
   { boot.loader.grub.enable = true;
-    boot.loader.grub.version = 2;
-    boot.loader.grub.device = "/dev/disk/by-id/${cfg.grubDeviceID}";
-
+    boot.loader.grub.device = if cfg.grubDeviceID == "nodev"  then "nodev" else "/dev/disk/by-id/${cfg.grubDeviceID}";
     boot.loader.timeout = 10;
+    # NixOS likes to fill up boot partitions with (by default) 100 old kernels.
+    # Keep a (for us) more reasonable number around.
+    boot.loader.grub.configurationLimit = 8;
     networking.firewall.enable = false;
 
     networking.hostId = cfg.hostId;

nixos/modules/README.rst

+These are mostly modelled on upstream nixos modules.
+They are generally fairly configurable (they don't tend to hard-code paths, they can be enabled or disabled).
+They don't know anything about morph (e.g. ``deployment.secrets``) or how the different grids are configured (e.g. ``grid.publicKeyPath``).
+Each module here tends to define one service (or group of related services) or feature.
+Eventually, all of these will be imported automatically and controlled by ``services.private-storage.*.enabled`` options.

nixos/modules/default.nix

+{
+  # Load modules that are sometimes universally useful and other times useful
+  # only for a specific service.  Where functionality is not universally
+  # useful, it needs to be enabled by a node's configuration.  By loading more
+  # modules (and therefore defining more options) than is strictly necessary
+  # for any single node the logic for supplying conditional configuration
+  # elsewhere is much simplified.  For example, a Vagrant module can
+  # unconditionally set up a filesystem for PaymentServer.  If PaymentServer
+  # is running on that node then it will get a Vagrant-appropriate
+  # configuration.  If PaymentServer hasn't been enabled then the
+  # configuration will just be ignored.
+  imports = [
+    ./packages.nix
+    ./issuer.nix
+    ./private-storage.nix
+    ./monitoring/policy.nix
+    ./monitoring/vpn/client.nix
+    ./monitoring/exporters/node.nix
+    ./monitoring/exporters/tahoe.nix
+    ./monitoring/exporters/promtail.nix
+  ];
+}

nixos/modules/deployment.nix

@@ -35,11 +35,11 @@ in {
   config = {
     # Configure the system to use our binary cache so that deployment updates
     # only require downloading pre-built software, not building it ourselves.
-    nix = {
-      binaryCachePublicKeys = [
+    nix.settings = {
+      trusted-public-keys = [
         "saxtons.private.storage:MplOcEH8G/6mRlhlKkbA8GdeFR3dhCFsSszrspE/ZwY="
       ];
-      binaryCaches = [
+      substituters = [
         "http://saxtons.private.storage"
       ];
     };

nixos/modules/issuer.nix

@@ -38,6 +38,15 @@ in {
         algorithm or Ristretto for Ristretto-flavored PrivacyPass.
       '';
     };
+    services.private-storage-issuer.tokensPerVoucher = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.int;
+      example = 50000;
+      description = ''
+        If not null, a value to pass to PaymentServer for
+        ``--tokens-per-voucher``.
+      '';
+    };
     services.private-storage-issuer.ristrettoSigningKeyPath = lib.mkOption {
       default = null;
       type = lib.types.path;
@@ -53,6 +62,13 @@ in {
         and payment management.
       '';
     };
+    services.private-storage-issuer.stripeWebhookSecretKeyPath = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        The path to a file containing a Stripe "webhook" secret key to use for
+        charge and payment management.
+      '';
+    };
     services.private-storage-issuer.stripeEndpointDomain = lib.mkOption {
       type = lib.types.str;
       description = ''
@@ -81,6 +97,15 @@ in {
         The kind of voucher database to use.
       '';
     };
+    services.private-storage-issuer.databaseFileSystem = lib.mkOption {
+      # Logically, the type is the type of an entry in fileSystems - but we'll
+      # just let the type system enforce that when we pass the value on to
+      # fileSystems.
+      description = ''
+        Configuration for a filesystem to mount which will hold the issuer's
+        internal state database.
+      '';
+    };
     services.private-storage-issuer.databasePath = lib.mkOption {
       default = null;
       type = lib.types.str;
@@ -114,7 +139,23 @@ in {
       # Payment server internal http port (arbitrary, non-priviledged):
       internalHttpPort = "1061";
 
+      # The "-vN" suffix indicates that this Nth incompatible version of on
+      # disk state as managed by this deployment system.  This does not have
+      # anything to do with what's inside the PaymentServer-managed state.
+      # Instead it's about things like the type of filesystem used or options
+      # having to do with the backing volume behind the filesystem.  In
+      # general I expect that to get from "-vN" to "-v(N+1)" some manual
+      # upgrade steps will be required.
+      stateDirectory = "zkapissuer-v2";
+
     in lib.mkIf cfg.enable {
+    # Make sure the voucher database filesystem is mounted.
+    fileSystems = {
+      "zkapissuer-data" = cfg.databaseFileSystem // {
+        mountPoint = "/var/lib/${stateDirectory}";
+      };
+    };
+
     # Add a systemd service to run PaymentServer.
     systemd.services.zkapissuer = {
       enable = true;
@@ -137,15 +178,30 @@ in {
       # Make systemd create a User/Group owned directory for PaymentServer
       # state. According to the docs at
       # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=
-      # "The specified directory names must be relative" ... this
-      # makes systemd create /var/lib/zkapissuer/ for us:
-      serviceConfig.StateDirectory = "zkapissuer";
+      # "The specified directory names must be relative" ... this makes
+      # systemd create this directory in /var/lib/ for us.
+      serviceConfig.StateDirectory = stateDirectory;
       serviceConfig.StateDirectoryMode = "0750";
 
+      unitConfig.AssertPathExists = [
         # Bail if there is still an old (root-owned) DB file on this system.
-      # If you hit this, and this /var/db/ file is indeed current, move it to
-      # /var/lib/zkapissuer/vouchers.sqlite3 and chown it to zkapissuer:zkapissuer.
-      unitConfig.AssertPathExists = "!/var/db/vouchers.sqlite3";
+        # If you hit this, and this /var/db/ file is indeed current, move it
+        # to /var/lib/zkapissuer/vouchers.sqlite3 and chown it to
+        # zkapissuer:zkapissuer.
+        "!/var/db/vouchers.sqlite3"
+
+        # Similarly, bail if the newer path you were just told to create --
+        # /var/lib/zkapissuer/vouchers.sqlite3 -- exists.  It needs to be
+        # moved /var/lib/zkapissuer-v2 where a dedicated filesystem has been
+        # created for it.
+        "!/var/lib/zkapissuer/vouchers.sqlite3"
+      ];
+
+      # Only start if the dedicated vouchers database filesystem is mounted so
+      # that we know we're going to find our vouchers database there (or that
+      # we will create it in the right place).
+      unitConfig.Requires = ["local-fs.target"];
+      unitConfig.After = ["local-fs.target"];
 
       script =
         let
@@ -167,11 +223,14 @@ in {
 
           stripeArgs =
             "--stripe-key-path ${cfg.stripeSecretKeyPath} " +
+            "--stripe-webhook-key-path ${cfg.stripeWebhookSecretKeyPath} " +
             "--stripe-endpoint-domain ${cfg.stripeEndpointDomain} " +
             "--stripe-endpoint-scheme ${cfg.stripeEndpointScheme} " +
             "--stripe-endpoint-port ${toString cfg.stripeEndpointPort}";
+
+          redemptionConfig = lib.optionalString (cfg.tokensPerVoucher != null) "--tokens-per-voucher ${builtins.toString cfg.tokensPerVoucher}";
         in
-          "${cfg.package.exePath} ${originArgs} ${issuerArgs} ${databaseArgs} ${httpArgs} ${stripeArgs}";
+          "${cfg.package.exePath} ${originArgs} ${issuerArgs} ${databaseArgs} ${httpArgs} ${stripeArgs} ${redemptionConfig}";
     };
 
     # PaymentServer runs as this user and group by default
@@ -195,7 +254,7 @@ in {
     ];
 
     # NGINX reverse proxy
-    security.acme.email = cfg.letsEncryptAdminEmail;
+    security.acme.defaults.email = cfg.letsEncryptAdminEmail;
     security.acme.acceptTerms = true;
     services.nginx = {
       enable = true;
@@ -214,6 +273,17 @@ in {
           # we pass less scanning spam on to our backend
           # Want a regex instead? try locations."~ /v\d+/"
           proxyPass = "http://127.0.0.1:${internalHttpPort}";
+
+          # The redemption endpoint can intentionally delay its response for
+          # up to 600 seconds for a cheap kind of server-push when payment
+          # completes.  Let that timeout control how long the connection stays
+          # open.  PaymentServer does not accept configuration for that
+          # duration so we also hard-code it here.
+          #
+          # http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout
+          extraConfig = ''
+            proxy_read_timeout 660;
+          '';
         };
         locations."/metrics" = {
           # Only allow our monitoringvpn subnet

nixos/modules/monitoring/exporters/megacli2prom.nix

@@ -12,10 +12,10 @@
 { config, options, lib, ourpkgs, pkgs, ... }:
 
 let
-  cfg = config.services.private-storage.monitoring.megacli2prom;
+  cfg = config.services.private-storage.monitoring.exporters.megacli2prom;
 
 in {
-  options.services.private-storage.monitoring.megacli2prom = {
+  options.services.private-storage.monitoring.exporters.megacli2prom = {
     enable = lib.mkEnableOption "MegaCli2Prom metrics gathering service";
     outFile = lib.mkOption {
       type = lib.types.str;
@@ -35,11 +35,10 @@ in {
 
   config =
     lib.mkIf cfg.enable {
-      environment.systemPackages = [ ourpkgs.megacli2prom ];
+      environment.systemPackages = [ ourpkgs.megacli2prom pkgs.megacli ];
       systemd.services.megacli2prom = {
         enable = true;
         description = "MegaCli2Prom metrics gathering service";
-        wantedBy = [ "multi-user.target" ];
         startAt = cfg.interval;
         path = [ pkgs.megacli ];
         # Save to a temp file and then move atomically so the

nixos/modules/monitoring/exporters/node.nix

@@ -6,15 +6,25 @@
 #        monitoring system
 # See https://nixos.org/manual/nixos/stable/#module-services-prometheus-exporters
 
-{ config, lib, pkgs, ... }:
+{ config, lib, pkgs, options, ... }:
 
 with lib;
 
 let
+  cfg = config.services.private-storage.monitoring.exporters.node;
   mountsFileSystemType = fsType: {} != filterAttrs (n: v: v.fsType == fsType) config.fileSystems;
 
 in {
-  config.services.prometheus.exporters.node = {
+  options.services.private-storage.monitoring.exporters.node = {
+    enable = lib.mkEnableOption "Base system metrics collection";
+    textfiles-directory = lib.mkOption {
+      type = lib.types.str;
+      description = "Directory used by the textfiles collector.";
+      default = "/run/prometheus-node-exporter";
+    };
+  };
+
+  config.services.prometheus.exporters.node = lib.mkIf cfg.enable {
     enable = true;
     openFirewall = true;
     firewallFilter = "-i monitoringvpn -p tcp -m tcp --dport 9100";
@@ -22,7 +32,7 @@ in {
     # extraFlags = [ "--collector.disable-defaults" ]; # not in nixpkgs 19.09
     # Thanks https://github.com/mayflower/nixexprs/blob/master/modules/monitoring/default.nix
     enabledCollectors = [
-      "arp"
+      # "arp" # is broken in 1.7.0 (2024-02-07)
       "bcache"
       "conntrack"
       "filefd"
@@ -30,16 +40,16 @@ in {
       "netclass"
       "netdev"
       "netstat"
-      #"rapl" # not in nixpkgs 19.09
+      "rapl"
       "sockstat"
-      #"softnet" # not in nixpkgs 19.09
+      "softnet"
       "stat"
       "systemd"
       "textfile"
-      "textfile.directory /run/prometheus-node-exporter"
-      #"thermal_zone" # not in nixpkgs 19.09
+      "textfile.directory ${cfg.textfiles-directory}"
+      "thermal_zone"
       "time"
-      #"udp_queues" # not in nixpkgs 19.09
+      "udp_queues"
       "uname"
       "vmstat"
     ] ++ optionals (!config.boot.isContainer) [
@@ -59,7 +69,7 @@ in {
     ] ++ (
       optionals (config.services.nfs.server.enable) [ "nfsd" ]
     ) ++ (
-      optionals ("" != config.boot.initrd.mdadmConf) [ "mdadm" ]
+      optionals ("" != config.boot.swraid.mdadmConf) [ "mdadm" ]
     ) ++ (
       optionals ({} != config.networking.bonds) [ "bonding" ]
     ) ++ (
@@ -67,7 +77,7 @@ in {
     ) ++ (
       optionals (mountsFileSystemType "xfs") [ "xfs" ]
     ) ++ (
-      optionals (mountsFileSystemType "zfs" || elem "zfs" config.boot.supportedFilesystems) [ "zfs" ]
+      optionals (mountsFileSystemType "zfs" || config.boot.supportedFilesystems.zfs or false) [ "zfs" ]
     );
   };
 }

nixos/modules/monitoring/exporters/promtail.nix

+# Promtail log forwarder configuration
+#
+# Scope: Tail logs on the local system and send them to Loki
+#
+# Description: This is not strictly an "exporter" like the Prometheus
+#              exporters, but it is very similar in what it is doing -
+#              preparing local data and sending it off to a TSDB.
+
+{ config, options, lib, ... }:
+
+let
+  cfg = config.services.private-storage.monitoring.exporters.promtail;
+  hostName = config.networking.hostName;
+  logRetention = toString(config.services.private-storage.monitoring.policy.logRetentionSeconds) + "s";
+
+in {
+  options.services.private-storage.monitoring.exporters.promtail = {
+    enable = lib.mkEnableOption "Promtail log exporter service";
+    lokiUrl = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The server URL that logs should be pushed to.
+      '';
+      # Resolving names is hard, let's have breakfast
+      # If you are curious why there's a plain IP address in here, read all of
+      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/251
+      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/257
+      # https://whetstone.private.storage/privatestorage/PrivateStorageio/-/merge_requests/258
+      default = "http://172.23.23.1:3100/loki/api/v1/push";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.promtail.enable = true;
+    networking.firewall.interfaces.monitoringvpn.allowedTCPPorts = [ 9080 ];
+
+    services.journald.extraConfig = ''
+      # This tells journald it can discard log files that contain only log
+      # entries older than...
+      MaxRetentionSec=${logRetention}
+
+      # This tells journald to start a new log file once a day.  Together with
+      # the MaxRetentionSec setting, this means that entries are kept for
+      # up to a full day longer than MaxRetentionSec.
+      #
+      # https://www.freedesktop.org/software/systemd/man/journald.conf.html
+      # for further details about these options.
+      #
+      MaxFileSec=1day
+
+      # This asks journald to not use more than 500M of disk space.  Due to
+      # journald's characteristics this might only be a week of logs, but that
+      # should be okay since we ship all logs to a central server that keeps
+      # them for a while longer.
+      SystemMaxUse=500M
+    '';
+
+    services.promtail.configuration = {
+      server = {
+        http_listen_port = 9080; # Using /metrics for health check
+        grpc_listen_address = "127.0.0.1"; # unused, but no option to turn it off.
+        grpc_listen_port = 9094; # unused, but no option to turn it off.
+      };
+
+      clients = [{
+          url = cfg.lokiUrl;
+      }];
+
+      scrape_configs = [{
+        job_name = "systemd-journal";
+        journal = {
+          labels = {
+            job = "systemd-journal";
+            host = hostName;
+          };
+        };
+        # The journal has many internal labels, that by default will
+        # be dropped because of their "__" prefix.  To keep them, rename them.
+        # https://grafana.com/docs/loki/latest/clients/promtail/scraping/#journal-scraping-linux-only
+        # https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+  };
+}

nixos/modules/monitoring/exporters/tahoe.nix

 # Tahoe Prometheus metrics collector
 #
-# Scope: Retrieves OpenMetrics from Tahoe and puts them
-#        where textfile collector can find them.
+# Scope: Retrieve metrics from Tahoe and put them where Prometheus'
+#        node-exporter's textfile collector can find them.
 #
 # Usage: Import this to every server running Tahoe.
 #
@@ -10,10 +10,11 @@
 { config, options, lib, pkgs, ... }:
 
 let
-  cfg = config.services.private-storage.monitoring.tahoe;
+  cfg = config.services.private-storage.monitoring.exporters.tahoe;
+  inherit (config.services.private-storage.monitoring.exporters.node) textfiles-directory;
 
 in {
-  options.services.private-storage.monitoring.tahoe = {
+  options.services.private-storage.monitoring.exporters.tahoe = {
     enable = lib.mkEnableOption "Tahoe OpenMetrics collecting service";
     scrapeEndpoint = lib.mkOption {
       type = lib.types.str;
@@ -23,7 +24,7 @@ in {
     outFile = lib.mkOption {
       type = lib.types.str;
       description = "Where to store the temporary file for node exporter to scrape?";
-      default = "/run/prometheus-node-exporter/tahoe.prom";
+      default = "${textfiles-directory}/tahoe.prom";
     };
     interval = lib.mkOption {
       type = lib.types.str;
@@ -38,6 +39,15 @@ in {
 
   config =
     lib.mkIf cfg.enable {
+      assertions = [
+        {
+          assertion = config.services.private-storage.monitoring.exporters.node.enable;
+          message = ''
+            services.private-storage.monitoring.tahoe requires services.private-storage.monitoring.exporters.node to provide the textfile prometheus collector.
+          '';
+        }
+      ];
+
       environment.systemPackages = [ pkgs.curl ];
 
       systemd.services.tahoe-metrics-collector = {
@@ -45,14 +55,22 @@ in {
         description = "Tahoe metrics gathering service";
         after = [ "tahoe.storage.service" ];
         startAt = cfg.interval;
-        path = [ pkgs.curl ];
+        path = [ pkgs.coreutils pkgs.findutils pkgs.curl ];
         restartIfChanged = false;
 
         # Save to a temp file and then move atomically so the
         # textfile collector won't read a partial file.
         # See https://github.com/prometheus/node_exporter#textfile-collector
         script = ''
-          curl --silent --show-error --fail-with-body --output "${cfg.outFile}.tmp" "${cfg.scrapeEndpoint}"
+          set -euo pipefail
+
+          NUM_CORRUPTION_ADVISORIES=$(find /storage/corruption-advisories/ -type f | wc -l)
+          echo "tahoe_corruption_advisories_total $NUM_CORRUPTION_ADVISORIES" > "${cfg.outFile}.tmp"
+
+          NUM_INCIDENT_REPORTS=$(find /var/db/tahoe-lafs/storage/logs/incidents/ -type f | wc -l)
+          echo "tahoe_incident_reports_total $NUM_INCIDENT_REPORTS" >> "${cfg.outFile}.tmp"
+
+          curl --silent --show-error --fail-with-body "${cfg.scrapeEndpoint}" >> "${cfg.outFile}.tmp"
           mv "${cfg.outFile}.tmp" "${cfg.outFile}"
         '';
       };

nixos/modules/monitoring/policy.nix

+# Codify our log data retention policy
+#
+# A maximum retention of 30 days conforms to the published log retention policy,
+# see https://private.storage/privacy-policy/ .
+
+{ options, lib, ... }: {
+  options.services.private-storage.monitoring.policy = {
+    logRetentionSeconds = lib.mkOption {
+      type = lib.types.int;
+      description = "How long do we retain logs (seconds)";
+      default = 29 * (24 * 60 * 60);  # 29 days, to accomodate for the journald log rotation (1 day).
+    };
+  };
+}