morph/grid/production/public-keys/users.nix

-let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGN4VQm3BIQKEFTw6aPrEwNuShf640N+Py2LOKznFCRT exarkun@bottom";
-in { "root" = key; "jcalderone" = key; }
+let
+  flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII6EUU/KNDr7y3m5OVWBZAuPiMJ4us3YOBEhxpG29yPN flo@la"];
+  last-resort = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1hy9mPkJI+7mY2Uq6CLpuFMMLOTfiY2sRJHwpihgRt cardno:26 269 859 - Last Resort A-Key"
+                  "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPYMUVNuWr2y+FL1GxW6S6jb3BWYhbzJ2zhvQVKu2ll cardno:23 845 763 - Last Resort C-key"
+                ];
+in {
+  "root" = flo ++ last-resort;
+  inherit  flo    last-resort;
+}

morph/grid/production/storage001-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/f72c1f46-6723-45bf-9ef7-92f31cc37589";
@@ -38,6 +36,6 @@
   } ];
 
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage002-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/0e92ada9-effb-42e2-a26a-9cdb529bcdc7";
@@ -37,6 +35,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage003-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, modulesPath, ... }:
 
 {
@@ -13,7 +11,7 @@
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
   boot.supportedFilesystems = [ "zfs" ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/240fc1f6-cd55-48a3-ac80-5b3550a32ef5";
@@ -38,6 +36,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage004-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/d628122e-05d9-4212-b6a5-4b9516d85dbe";
@@ -32,6 +30,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 32;
+  nix.settings.max-jobs = lib.mkDefault 32;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage005-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/2653c6bb-396f-4911-b9ff-b68de8f9715d";
@@ -37,6 +35,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 32;
+  nix.settings.max-jobs = lib.mkDefault 32;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/testing/config.json

@@ -16,7 +16,6 @@
 , "allowedChargeOrigins": [
     "http://localhost:5000"
   , "https://privatestorage-staging.com"
-  , "https://www.privatestorage-staging.com"
   ]
 , "monitoringGoogleOAuthClientID": "802959152038-6esn1c6u2lm3j82lf29jvmn8s63hi8dc.apps.googleusercontent.com"
 }

morph/grid/testing/grid.nix

 # See morph/grid/local/grid.nix for additional commentary.
 let
-  pkgs = import <nixpkgs> { };
-
   gridlib = import ../../lib;
-  grid-config = pkgs.lib.trivial.importJSON ./config.json;
+  grid-config = builtins.fromJSON (builtins.readFile ./config.json);
 
   # Module with per-grid configuration
   grid-module = {config, ...}: {
@@ -72,14 +70,14 @@ let
         inherit paymentExporterTargets blackboxExporterHttpsTargets;
         inherit (grid-config) monitoringDomains;
         googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
-        enableSlackAlert = true;
+        enableZulipAlert = true;
       };
       system.stateVersion = "19.09";
     };
   };
 
   # TBD: derive these automatically:
-  paymentExporterTargets = [ "payments" ];
+  paymentExporterTargets = [ "payments.monitoringvpn" ];
   blackboxExporterHttpsTargets = [
     "https://privatestorage-staging.com/"
     "https://www.privatestorage-staging.com/"

morph/grid/testing/public-keys/borgbackup/storage001.repopath

+p2kt6691@p2kt6691.repo.borgbase.com:repo

morph/grid/testing/public-keys/users.nix

 let
-  jcalderone = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
-  flo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
+  flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII78HGtpjFxQo7wol85hqfoCqjdK9Nk7+82rwttyLHpe flo@la-staging"];
+  bdonneaux = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGgpTXgxEqQPSl17NzJkAJgeDSFS1Ke/qjCuVMTZLlna benoit@leastauthority.com" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIZtWY7t8HVnaz6bluYsrAlzZC3MZtb8g0nO5L5fCQKR benoit@leastauthority.com"];
+  chris = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT3/sSNoJP3E17oFTYjHN+uOwTY1wVox9tff97iueIo4V88eAc/oXKETiwPkr33qbGwuoXKkxIXJVz8rnNtO6IjTm9qfgzPiRAUJjew8bunL+V7SbhQIv7nk1fyV/efaENElG8bdmzTEpgwGcEnyibqvHJSYX6W+dMCz3G1t/lv97if3ohZKENHuMC5hLfJbGHSGKFO5XdjEjeda9lDd9Ac8XyruaL7iqEefsC7GuUgNRn8V83vwuJMDAC2xXC2V11M65VkGs6WPAct2+llzTtYbsxjxVZXC4yU42eXJYfBZEcCTPtJsKJxQCqSgFOEUnOYiuS6p4Q7a97BfHJ9S9oOV8U/e7YeE4b9Q8TPNzvKTPBAsuKyLyNYekBDB7fOTFziuJy/L578EaDv2BxrsfyCQqtjLko6TIAUbbHvce8urWNvj7H+fNXaURLIQmSTOv/mMl+omkvbP3MNgSFdENpCZaHSTiDxjygf52xcinj6Ijf3uDvPY2UjIRrbWSNV4MYpZDfkqt9THY4QibmxhER/YGvY+0zfiYGqQpQMMbTUB9hhoO5AHnnhMszNG2V9i70VqWyEMsS+Sr1+gOVAPraLp/tqHaqZk7/c4DDpILjA+4davTL6lgaiewx8a0ZEPAKZCZkOMovZKwkVIjyMvfekUkf1cF+QigJPZzcWSWEjQ== cardno:000608671823"];
+  last-resort = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1hy9mPkJI+7mY2Uq6CLpuFMMLOTfiY2sRJHwpihgRt cardno:26 269 859 - Last Resort A-Key"
+                  "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPYMUVNuWr2y+FL1GxW6S6jb3BWYhbzJ2zhvQVKu2ll cardno:23 845 763 - Last Resort C-key"
+                ];
 in
   {
-    "root" = jcalderone;
-    inherit jcalderone;
-    inherit flo;
+    "root" = flo ++ bdonneaux ++ chris ++ last-resort;
+    inherit  flo    bdonneaux    chris    last-resort;
   }

morph/lib/base.nix

@@ -43,6 +43,8 @@
   # Any extra NixOS modules to load on all our servers.  Note that just
   # because they're loaded doesn't *necessarily* mean they're turned on.
   imports = [
+    # Set options intended for a "small" NixOS: Do not build X and docs.
+    <nixpkgs/nixos/modules/profiles/minimal.nix>
     # This brings in various other modules that define options for different
     # areas of the service.
     ../../nixos/modules/default.nix
@@ -57,6 +59,19 @@
     # qualified domain name.
     deployment.targetHost = config.networking.fqdn;
 
+    services.private-storage.monitoring.exporters.promtail.enable = true;
+
+    # Install no documentation on grid
+    # It seems 24.05 has some new defaults that aren't stripped away by the
+    # 'minimal' profile above.
+    # See https://github.com/NixOS/nixpkgs/blob/nixos-24.05/nixos/modules/misc/documentation.nix
+    documentation.enable = false;
+    documentation.man.enable = false;
+    documentation.info.enable = false;
+    documentation.doc.enable = false;
+    documentation.dev.enable = false;
+    documentation.nixos.enable = false;
+
     assertions = [
       # This is a check to save somebody in the future trying to debug why
       # setting `nixpkgs.config` is not having an effect.
@@ -65,7 +80,7 @@
         assertion = config.nixpkgs.config == {};
         message = ''
           Since we set `nixpkgs.pkgs` via morph's `network.pkgs`, the value for `nixpkgs.config` is ignored.
-          See https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/85#note_15876 for details.
+          See https://whetstone.private.storage/privatestorage/PrivateStorageio/-/issues/85#note_15876 for details.
           '';
       }
     ];

morph/lib/bootstrap-configuration.nix

@@ -67,7 +67,7 @@ let
   # Stop!  I hope you're done when you get here.  If you have to modify
   # anything below this point the expression should probably be refactored and
   # another variable added controlling whatever new thing you need to control.
-  # Open an issue: https://whetstone.privatestorage.io/privatestorage/PrivateStorageio/-/issues/new?issue
+  # Open an issue: https://whetstone.private.storage/privatestorage/PrivateStorageio/-/issues/new?issue
 in
 # Define a function that ignores all its arguments.  We don't need any of them
 # for now.
@@ -87,7 +87,6 @@ in
   # Configure the bootloader how we like.
   boot.loader.timeout = 10;
   boot.loader.grub.enable = true;
-  boot.loader.grub.version = 2;
   boot.loader.grub.device = "/dev/disk/by-id/${grubDeviceID}";
 
   # Let me in to do subsequent configuration.  This makes the machine wide

morph/lib/borgbackup.nix

+# Importing this adds a daily borg backup job to a node.
+# It has all the common config and keys, and can be configured
+# to back up more (or entirely different) folders.
+
+
+{ lib, config, pkgs, ...}:
+let
+  cfg = config.services.private-storage.borgbackup;
+  inherit (config.grid) publicKeyPath privateKeyPath;
+
+  # For each host generate a number between 0 and 15 so backup
+  # jobs don't all run at the same time.
+  ip-util = import ../../nixos/lib/ip-util.nix;
+  backupDelay = with builtins; bitAnd (ip-util.fromHexString
+    (substring 0 6 (hashString "md5" config.networking.hostName))) 15;
+
+in {
+  options.services.private-storage.borgbackup = {
+    enable = lib.mkEnableOption "Borgbackup daily backup job";
+    paths = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of directories to back up using Borg.
+      '';
+      default = [ "/storage" ];
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    deployment = {
+      secrets = {
+        "borgbackup-passphrase" = {
+          # The passphrase is used to encrypt the repo key
+          # https://borgbackup.readthedocs.io/en/stable/usage/init.html
+          destination = "/run/keys/borgbackup/passphrase";
+          source = "${privateKeyPath}/borgbackup.passphrase";
+        };
+        "borgbackup-appendonly-ssh-key" = {
+          # The ssh key is used to authenticate to the remote repo server
+          destination = "/run/keys/borgbackup/ssh-key";
+          source = "${privateKeyPath}/borgbackup.ssh-key";
+        };
+      };
+    };
+
+    services.borgbackup.jobs = {
+      daily = {
+        paths = cfg.paths;
+        repo = lib.fileContents "${publicKeyPath}/borgbackup/${config.networking.hostName}.repopath";
+        doInit = false;
+        encryption = {
+          mode = "repokey-blake2";
+          passCommand = "cat /run/keys/borgbackup/passphrase";
+        };
+        environment = {
+          BORG_RSH = "ssh -i /run/keys/borgbackup/ssh-key -o StrictHostKeyChecking=accept-new";
+        };
+
+	# Output statistics after uploading a backup set
+	extraCreateArgs = "--stats --json";
+	# All logs in JSON to help Prometheus/Grafana
+	extraArgs = "--log-json";
+
+        # Ciphertext doesn't compress well
+        compression = "none";
+
+        # Start the backup at a different time per machine,
+        # and not at the full hour, but somewhat later
+        startAt = "*-*-* " + toString backupDelay + ":22:11 UTC";
+      };
+    };
+
+    # Check repo once a month
+    systemd.services.borgbackup-check-repo = {
+      # Once a month, 3h after last backup started.
+      # Add "1" because day 0 is invalid.
+      startAt = "*-*-" + toString (backupDelay + 1) + " 18:33:22 UTC";
+      path = [ pkgs.borgbackup ];
+      environment = {
+        BORG_PASSCOMMAND = "cat /run/keys/borgbackup/passphrase";
+        BORG_RSH = "ssh -i /run/keys/borgbackup/ssh-key -o StrictHostKeyChecking=accept-new";
+        BORG_REPO = lib.fileContents "${publicKeyPath}/borgbackup/${config.networking.hostName}.repopath";
+      };
+      script = ''${pkgs.borgbackup}/bin/borg check --verbose --log-json'';
+    };
+  };
+}

morph/lib/default.nix

@@ -6,10 +6,13 @@
 
   hardware-aws = import ./issuer-aws.nix;
   hardware-vagrant = import ./hardware-vagrant.nix;
+  hardware-monitoring-ovh = import ./issuer-monitoring-ovh.nix;
+  hardware-payments-ovh = import ./issuer-payments-ovh.nix;
 
   issuer = import ./issuer.nix;
   storage = import ./storage.nix;
   monitoring = import ./monitoring.nix;
+  borgbackup = import ./borgbackup.nix;
 
   modules = builtins.toString ../../nixos/modules;
 

morph/lib/hardware-vagrant.nix

@@ -15,18 +15,19 @@
   };
 
   config = {
-    virtualisation.virtualbox.guest.enable = true;
+    services.qemuGuest.enable = true;
 
-    boot.loader.grub.device = "/dev/sda";
+    boot.loader.grub.device = "/dev/vda";
 
-    boot.initrd.availableKernelModules = [ "ata_piix" "sd_mod" "sr_mod" ];
+    boot.initrd.availableKernelModules = [ "ata_piix" "virtio_pci" "virtio_blk" "sd_mod" "sr_mod" ];
     boot.kernel.sysctl = { "vm.swappiness" = 0; };
+    boot.kernelParams = [ "console=tty0" "console=ttyS0,115200" ];
 
     # remove the fsck that runs at startup. It will always fail to run, stopping
     # your boot until you press *.
     boot.initrd.checkJournalingFS = false;
 
-    networking.interfaces.enp0s8.ipv4.addresses = [{
+    networking.interfaces.ens5.ipv4.addresses = [{
       address = config.grid.publicIPv4;
       prefixLength = 24;
     }];
@@ -47,11 +48,11 @@
     fileSystems."/storage" = { fsType = "tmpfs"; };
 
     fileSystems."/" =
-      { device = "/dev/sda1";
+      { device = "/dev/vda1";
         fsType = "ext4";
       };
 
     # We want to push packages with morph without having to sign them
-    nix.trustedUsers = [ "@wheel" "root" "vagrant" ];
+    nix.settings.trusted-users = [ "@wheel" "root" "vagrant" ];
   };
 }

morph/lib/issuer-aws.nix

@@ -6,13 +6,23 @@
   boot.loader.grub.device = lib.mkForce "/dev/nvme0n1";
 
   ec2.hvm = true;
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
   swapDevices = [ {
     device = "/var/swapfile";
-    size = 4096; # megabytes
+    size = 1024; # megabytes
     randomEncryption = true;
   } ];
 
+  # If we don't manually and explicitly early-load the loop module, crypt-swap
+  # setup fails with the not very helpful message: "loop device with autoclear
+  # flag is required"
+  # See https://unix.stackexchange.com/a/554500/81275
+  boot.kernelModules = [ "loop" ];
+
+  # NixOS likes to fill up boot partitions with (by default) 100 old kernels.
+  # Keep a (for us) more reasonable number around.
+  boot.loader.grub.configurationLimit = 8;
+
   # Break the tie between AWS and morph for the hostname by forcing the
   # morph-supplied name.  See also
   # <https://github.com/DBCDK/morph/issues/146>.
@@ -32,4 +42,12 @@
     dates = "weekly";
     options = "--delete-older-than 30d";
   };
+
+  # Turn on automatic optimization of nix store
+  # https://nixos.wiki/wiki/Storage_optimization
+  nix.settings.auto-optimise-store = true;
+
+  # Most of the time, we have ample free & usable memory, but when upgrading
+  # software, we sometimes run out because of Nix.  This is supposed to help:
+  zramSwap.enable = true;
 }

morph/lib/issuer-monitoring-ovh.nix

+{ modulesPath, name, lib, ... }: {
+
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+  boot.loader.grub.device = "/dev/sda";
+  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" ];
+  boot.initrd.kernelModules = [ "nvme" ];
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
+
+  fileSystems."/" = { device = "/dev/sda1"; fsType = "ext4"; };
+
+  swapDevices = [ {
+    device = "/var/swapfile";
+    size = 4096; # megabytes
+    randomEncryption = true;
+  } ];
+
+  # Break the tie between AWS and morph for the hostname by forcing the
+  # morph-supplied name.  See also
+  # <https://github.com/DBCDK/morph/issues/146>.
+  networking.hostName = name;
+
+  # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+  # beyond control of this particular part of the system) for the
+  # PaymentServer voucher database.  This makes it easier to manage for
+  # tasks like backup/recovery and encryption.
+  services.private-storage-issuer.databaseFileSystem = {
+    label = "zkapissuer-data";
+  };
+
+  # Clean up packages after a while
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 30d";
+  };
+
+  # Turn on automatic optimization of nix store
+  # https://nixos.wiki/wiki/Storage_optimization
+  nix.settings.auto-optimise-store = true;
+}

morph/lib/issuer-payments-ovh.nix

+{ modulesPath, name, lib, ... }: {
+
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+  boot.loader.grub.device = "/dev/sda";
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
+
+  fileSystems."/" = { device = "/dev/sda3"; fsType = "ext4"; };
+  
+  swapDevices = [ {
+    device = "/dev/sda2";
+    randomEncryption = true;
+  } ];
+
+  # Break the tie between AWS and morph for the hostname by forcing the
+  # morph-supplied name.  See also
+  # <https://github.com/DBCDK/morph/issues/146>.
+  networking.hostName = name;
+
+  # Mount a dedicated filesystem (ideally on a dedicated volume, but that's
+  # beyond control of this particular part of the system) for the
+  # PaymentServer voucher database.  This makes it easier to manage for
+  # tasks like backup/recovery and encryption.
+  services.private-storage-issuer.databaseFileSystem = {
+    label = "zkapissuer-data";
+  };
+
+  # Clean up packages after a while
+  nix.gc = {
+    automatic = true;
+    dates = "weekly";
+    options = "--delete-older-than 30d";
+  };
+
+  # Turn on automatic optimization of nix store
+  # https://nixos.wiki/wiki/Storage_optimization
+  nix.settings.auto-optimise-store = true;
+}

morph/lib/issuer.nix

@@ -2,12 +2,11 @@
 # "issuer"-type system.
 { lib, config, ...}:
 let
-  inherit (config.grid) publicKeyPath privateKeyPath monitoringvpnEndpoint monitoringvpnIPv4;
-  inherit (config.grid.issuer) issuerDomains allowedChargeOrigins;
+  inherit (config.grid) privateKeyPath;
+  inherit (config.grid.issuer) issuerDomains allowedChargeOrigins tokensPerVoucher;
 in {
   imports = [
-    ../../nixos/modules/monitoring/vpn/client.nix
-    ../../nixos/modules/monitoring/exporters/node.nix
+    ./monitoringvpn-client.nix
   ];
 
   options.grid.issuer = {
@@ -26,6 +25,16 @@ in {
         to allow.
       '';
     };
+
+    tokensPerVoucher = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.int;
+      example = 50000;
+      description = ''
+        If not null, a value to pass to PaymentServer for
+        ``--tokens-per-voucher``.
+      '';
+    };
   };
 
   config = {
@@ -47,44 +56,31 @@ in {
           permissions = "0400";
           action = ["sudo" "systemctl" "restart" "zkapissuer.service"];
         };
-
-        "monitoringvpn-secret-key" = {
-          destination = "/run/keys/monitoringvpn/client.key";
-          source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
-          owner.user = "root";
-          owner.group = "root";
-          permissions = "0400";
-          action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
-        };
-        "monitoringvpn-preshared-key" = {
-          destination = "/run/keys/monitoringvpn/preshared.key";
-          source = "${privateKeyPath}/monitoringvpn/preshared.key";
-          owner.user = "root";
-          owner.group = "root";
+        "stripe-webhook-secret-key" = {
+          destination = "/run/keys/stripe.webhook-secret-key";
+          source = "${privateKeyPath}/stripe.webhook-secret";
+          owner.user = "zkapissuer";
+          owner.group = "zkapissuer";
           permissions = "0400";
-          action = ["sudo" "systemctl" "restart" "wireguard-monitoringvpn.service"];
+          action = ["sudo" "systemctl" "restart" "zkapissuer.service"];
         };
       };
     };
-
     services.private-storage-issuer = {
       enable = true;
       tls = true;
       ristrettoSigningKeyPath = config.deployment.secrets.ristretto-signing-key.destination;
       stripeSecretKeyPath = config.deployment.secrets.stripe-secret-key.destination;
+      stripeWebhookSecretKeyPath = config.deployment.secrets.stripe-webhook-secret-key.destination;
       database = "SQLite3";
       databasePath = "${config.fileSystems."zkapissuer-data".mountPoint}/vouchers.sqlite3";
       inherit (config.grid) letsEncryptAdminEmail;
       inherit allowedChargeOrigins;
       domains = issuerDomains;
+      inherit tokensPerVoucher;
     };
 
-    services.private-storage.monitoring.vpn.client = {
-      enable = true;
-      ip = monitoringvpnIPv4;
-      endpoint = monitoringvpnEndpoint;
-      endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
-    };
+    services.private-storage.monitoring.exporters.node.enable = true;
 
     system.stateVersion = "19.03";
   };

morph/lib/monitoring.nix

@@ -24,16 +24,14 @@ let
   vpnClientIPs = lib.remove monitoringvpnIPv4 (map (node: node.vpnIPv4) monitoringHosts);
   # A list of VPN clients (IP addresses or hostnames) as strings indicating
   # which nodes to scrape "nodeExporter" metrics from.
-  nodeExporterTargets = map (node: node.name) monitoringHosts;
+  nodeExporterTargets = map (node: node.vpnHostName) monitoringHosts;
 in {
   imports = [
     ../../nixos/modules/monitoring/vpn/server.nix
     ../../nixos/modules/monitoring/server/grafana.nix
     ../../nixos/modules/monitoring/server/prometheus.nix
-    ../../nixos/modules/monitoring/exporters/node.nix
+    ../../nixos/modules/monitoring/server/loki.nix
     ../../nixos/modules/monitoring/exporters/blackbox.nix
-    # Loki 0.3.0 from Nixpkgs 19.09 is too old and does not work:
-    # ../../nixos/modules/monitoring/server/loki.nix
   ];
 
   options.grid.monitoring = {
@@ -79,6 +77,15 @@ in {
         When true requires a grafana-slack-url file (see private-keys/README.rst).
       '';
     };
+
+    enableZulipAlert = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to enable alerting via Zulip.
+        When true requires a grafana-zulip-url file (see private-keys/README.rst).
+      '';
+    };
   };
 
   config = {
@@ -140,6 +147,16 @@ in {
           action = ["sudo" "systemctl" "restart" "grafana.service"];
         };
       })
+      (lib.mkIf cfg.enableZulipAlert {
+        "grafana-zulip-url" = {
+          source = "${privateKeyPath}/grafana-zulip-url";
+          destination = "/run/keys/grafana-zulip-url";
+          owner.user = config.systemd.services.grafana.serviceConfig.User;
+          owner.group = config.users.users.grafana.group;
+          permissions = "0400";
+          action = ["sudo" "systemctl" "restart" "grafana.service"];
+        };
+      })
     ];
 
     networking.hosts = hostsMap;
@@ -158,9 +175,11 @@ in {
     };
 
     services.private-storage.monitoring.grafana = {
-      inherit (cfg) googleOAuthClientID enableSlackAlert ;
+      inherit (cfg) googleOAuthClientID enableSlackAlert enableZulipAlert;
       inherit letsEncryptAdminEmail;
       domains = cfg.monitoringDomains;
     };
+
+    services.private-storage.monitoring.exporters.node.enable = true;
   };
 }