morph/grid/production/public-keys/borgbackup/storage004.repopath

+sbn13vf8@sbn13vf8.repo.borgbase.com:repo

morph/grid/production/public-keys/borgbackup/storage005.repopath

+wg8x4po7@wg8x4po7.repo.borgbase.com:repo

morph/grid/production/public-keys/deploy_key.pub

+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK50RwXncelNB4JAazoXEhCxXbJZ79qWcQMAWeX14H+W exarkun@baryon

morph/grid/production/public-keys/users.nix

-let key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGN4VQm3BIQKEFTw6aPrEwNuShf640N+Py2LOKznFCRT exarkun@bottom";
-in { "root" = key; "jcalderone" = key; }
+let
+  flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII6EUU/KNDr7y3m5OVWBZAuPiMJ4us3YOBEhxpG29yPN flo@la"];
+  last-resort = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1hy9mPkJI+7mY2Uq6CLpuFMMLOTfiY2sRJHwpihgRt cardno:26 269 859 - Last Resort A-Key"
+                  "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPYMUVNuWr2y+FL1GxW6S6jb3BWYhbzJ2zhvQVKu2ll cardno:23 845 763 - Last Resort C-key"
+                ];
+in {
+  "root" = flo ++ last-resort;
+  inherit  flo    last-resort;
+}

morph/grid/production/storage001-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/f72c1f46-6723-45bf-9ef7-92f31cc37589";
@@ -38,6 +36,6 @@
   } ];
 
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage002-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/0e92ada9-effb-42e2-a26a-9cdb529bcdc7";
@@ -37,6 +35,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage003-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, modulesPath, ... }:
 
 {
@@ -13,7 +11,7 @@
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
   boot.supportedFilesystems = [ "zfs" ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/240fc1f6-cd55-48a3-ac80-5b3550a32ef5";
@@ -38,6 +36,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 24;
+  nix.settings.max-jobs = lib.mkDefault 24;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage004-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/d628122e-05d9-4212-b6a5-4b9516d85dbe";
@@ -32,6 +30,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 32;
+  nix.settings.max-jobs = lib.mkDefault 32;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/production/storage005-hardware.nix

-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
+# NixOS configuration specific to this node
 { config, lib, pkgs, ... }:
 
 {
@@ -12,7 +10,7 @@
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
+  boot.kernel.sysctl = { "vm.swappiness" = 1; };
 
   fileSystems."/" =
     { device = "/dev/disk/by-uuid/2653c6bb-396f-4911-b9ff-b68de8f9715d";
@@ -37,6 +35,6 @@
     randomEncryption = true;
   } ];
 
-  nix.maxJobs = lib.mkDefault 32;
+  nix.settings.max-jobs = lib.mkDefault 32;
   powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }

morph/grid/testing/config.json

@@ -8,10 +8,14 @@
     "payments.privatestorage-staging.com"
   , "payments.extra.privatestorage-staging.com"
   ]
+, "monitoringDomains": [
+    "monitoring.privatestorage-staging.com"
+  , "monitoring.extra.privatestorage-staging.com"
+  ]
 , "letsEncryptAdminEmail": "jean-paul@privatestorage.io"
 , "allowedChargeOrigins": [
     "http://localhost:5000"
   , "https://privatestorage-staging.com"
-  , "https://www.privatestorage-staging.com"
   ]
+, "monitoringGoogleOAuthClientID": "802959152038-6esn1c6u2lm3j82lf29jvmn8s63hi8dc.apps.googleusercontent.com"
 }

morph/grid/testing/grid.nix

 # See morph/grid/local/grid.nix for additional commentary.
 let
-  pkgs = import <nixpkgs> { };
-
   gridlib = import ../../lib;
-  rawConfig = pkgs.lib.trivial.importJSON ./config.json;
-  config = rawConfig // {
-    sshUsers = import ./public-keys/users.nix;
+  grid-config = builtins.fromJSON (builtins.readFile ./config.json);
 
+  # Module with per-grid configuration
+  grid-module = {config, ...}: {
+    imports = [
+      gridlib.base
+      # Allow us to remotely trigger updates to this system.
+      ../../../nixos/modules/deployment.nix
+      # Give it a good SSH configuration.
+      ../../../nixos/modules/ssh.nix
+    ];
+    services.private-storage.sshUsers = import ./public-keys/users.nix;
+    networking.domain = grid-config.domain;
     # Convert relative paths to absolute so library code can resolve names
     # correctly.
-    publicKeyPath = toString ./. + "/${rawConfig.publicKeyPath}";
-    privateKeyPath = toString ./. + "/${rawConfig.privateKeyPath}";
+    grid = {
+      publicKeyPath = toString ./. + "/${grid-config.publicKeyPath}";
+      privateKeyPath = toString ./. + "/${grid-config.privateKeyPath}";
+      inherit (grid-config) monitoringvpnEndpoint letsEncryptAdminEmail;
+    };
+    # Configure deployment management authorization for all systems in the grid.
+    services.private-storage.deployment = {
+      authorizedKey = builtins.readFile "${config.grid.publicKeyPath}/deploy_key.pub";
+      gridName = "testing";
+    };
   };
 
-  payments = {name, ...}: {
-    networking.hostName = name;
+  payments = {
     imports = [
       gridlib.issuer
       gridlib.hardware-aws
-      (gridlib.customize-issuer (config // {
-        monitoringvpnIPv4 = "172.23.23.11";
-      }))
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.11";
+      grid.issuer = {
+        inherit (grid-config) issuerDomains allowedChargeOrigins;
+      };
+    };
   };
 
-  storage001 = {name, ...}: {
-    networking.hostName = name;
+  storage001 = {
     imports = [
       gridlib.storage
+      gridlib.hardware-aws
       ./testing001-hardware.nix
-      (gridlib.customize-storage (config // {
-        monitoringvpnIPv4 = "172.23.23.12";
-        stateVersion = "19.03";
-      }))
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.12";
+      grid.storage = {
+        inherit (grid-config) passValue publicStoragePort;
+      };
+      system.stateVersion = "19.03";
+    };
   };
 
-  monitoring = {name, ...}: {
-    networking.hostName = name;
+  monitoring = {
     imports = [
       gridlib.monitoring
       gridlib.hardware-aws
-      (gridlib.customize-monitoring {
-        inherit hostsMap vpnClientIPs nodeExporterTargets paymentExporterTargets;
-        inherit (config) domain publicKeyPath privateKeyPath;
-        monitoringvpnIPv4 = "172.23.23.1";
-        stateVersion = "19.09";
-      })
+      grid-module
     ];
+    config = {
+      grid.monitoringvpnIPv4 = "172.23.23.1";
+      grid.monitoring = {
+        inherit paymentExporterTargets blackboxExporterHttpsTargets;
+        inherit (grid-config) monitoringDomains;
+        googleOAuthClientID = grid-config.monitoringGoogleOAuthClientID;
+        enableZulipAlert = true;
+      };
+      system.stateVersion = "19.09";
+    };
   };
 
   # TBD: derive these automatically:
-  hostsMap = {
-    "172.23.23.1"  = [ "monitoring" "monitoring.monitoringvpn" ];
-    "172.23.23.11" = [ "payments"   "payments.monitoringvpn"   ];
-    "172.23.23.12" = [ "storage001" "storage001.monitoringvpn" ];
-  };
-  vpnClientIPs = [ "172.23.23.11" "172.23.23.12" ];
-  nodeExporterTargets = [ "monitoring" "payments" "storage001" ];
-  paymentExporterTargets = [ "payments" ];
+  paymentExporterTargets = [ "payments.monitoringvpn" ];
+  blackboxExporterHttpsTargets = [
+    "https://privatestorage-staging.com/"
+    "https://www.privatestorage-staging.com/"
+    "https://extra.privatestorage-staging.com/"
+    "https://www.extra.privatestorage-staging.com/"
+    "https://payments.privatestorage-staging.com/"
+    "https://payments.extra.privatestorage-staging.com/"
+    "https://monitoring.privatestorage-staging.com/"
+    "https://monitoring.extra.privatestorage-staging.com/"
+  ];
 
 in {
   network = {
     description = "PrivateStorage.io Testing Grid";
+    inherit (gridlib) pkgs;
   };
   inherit payments monitoring storage001;
 }

morph/grid/testing/public-keys/borgbackup/storage001.repopath

+p2kt6691@p2kt6691.repo.borgbase.com:repo

morph/grid/testing/public-keys/deploy_key.pub

+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB88qfLdoR5Pq9Us7vOVc6wBWmIDxme9MXYQSxxO+8/X exarkun@baryon

morph/grid/testing/public-keys/users.nix

 let
-  jcalderone = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN4GenAY/YLGuf1WoMXyyVa3S9i4JLQ0AG+pt7nvcLlQ exarkun@baryon";
-  flo = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHx7wJQNqKn8jOC4AxySRL2UxidNp7uIK9ad3pMb1ifF flo@fs-la";
+  flo = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII78HGtpjFxQo7wol85hqfoCqjdK9Nk7+82rwttyLHpe flo@la-staging"];
+  bdonneaux = ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGgpTXgxEqQPSl17NzJkAJgeDSFS1Ke/qjCuVMTZLlna benoit@leastauthority.com" "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIZtWY7t8HVnaz6bluYsrAlzZC3MZtb8g0nO5L5fCQKR benoit@leastauthority.com"];
+  chris = ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT3/sSNoJP3E17oFTYjHN+uOwTY1wVox9tff97iueIo4V88eAc/oXKETiwPkr33qbGwuoXKkxIXJVz8rnNtO6IjTm9qfgzPiRAUJjew8bunL+V7SbhQIv7nk1fyV/efaENElG8bdmzTEpgwGcEnyibqvHJSYX6W+dMCz3G1t/lv97if3ohZKENHuMC5hLfJbGHSGKFO5XdjEjeda9lDd9Ac8XyruaL7iqEefsC7GuUgNRn8V83vwuJMDAC2xXC2V11M65VkGs6WPAct2+llzTtYbsxjxVZXC4yU42eXJYfBZEcCTPtJsKJxQCqSgFOEUnOYiuS6p4Q7a97BfHJ9S9oOV8U/e7YeE4b9Q8TPNzvKTPBAsuKyLyNYekBDB7fOTFziuJy/L578EaDv2BxrsfyCQqtjLko6TIAUbbHvce8urWNvj7H+fNXaURLIQmSTOv/mMl+omkvbP3MNgSFdENpCZaHSTiDxjygf52xcinj6Ijf3uDvPY2UjIRrbWSNV4MYpZDfkqt9THY4QibmxhER/YGvY+0zfiYGqQpQMMbTUB9hhoO5AHnnhMszNG2V9i70VqWyEMsS+Sr1+gOVAPraLp/tqHaqZk7/c4DDpILjA+4davTL6lgaiewx8a0ZEPAKZCZkOMovZKwkVIjyMvfekUkf1cF+QigJPZzcWSWEjQ== cardno:000608671823"];
+  last-resort = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1hy9mPkJI+7mY2Uq6CLpuFMMLOTfiY2sRJHwpihgRt cardno:26 269 859 - Last Resort A-Key"
+                  "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPYMUVNuWr2y+FL1GxW6S6jb3BWYhbzJ2zhvQVKu2ll cardno:23 845 763 - Last Resort C-key"
+                ];
 in
   {
-    "root" = jcalderone;
-    inherit jcalderone;
-    inherit flo;
+    "root" = flo ++ bdonneaux ++ chris ++ last-resort;
+    inherit  flo    bdonneaux    chris    last-resort;
   }

morph/grid/testing/testing001-hardware.nix

+{ lib, ... }:
 {
-  imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];
-  ec2.hvm = true;
-  boot.kernel.sysctl = { "vm.swappiness" = 0; };
-  swapDevices = [ {
-    device = "/var/swapfile";
-    size = 8192; # megabytes
-    randomEncryption = true;
-  } ];
-
-
   boot.supportedFilesystems = [ "zfs" ];
   networking.hostId = "10000000";
 
   # Manually created using:
-  #   zpool create -m legacy -o ashift=12 root raidz /dev/disk/by-id/{nvme-nvme.1d0f-766f6c3038623133353836383465643436363430-416d617a6f6e20456c617374696320426c6f636b2053746f7265-00000001,nvme-nvme.1d0f-766f6c3034653531383066303134633436653034-416d617a6f6e20456c617374696320426c6f636b2053746f7265-00000001,nvme-nvme.1d0f-766f6c3062333164633831386366623231373730-416d617a6f6e20456c617374696320426c6f636b2053746f7265-00000001,nvme-nvme.1d0f-766f6c3061353939623438336661353933636664-416d617a6f6e20456c617374696320426c6f636b2053746f7265-00000001}
+  #   zpool create -m legacy -o ashift=12 root /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol047fc3ea6bafdcc58
   fileSystems."/storage" = {
     device = "root";
     fsType = "zfs";
   };
+
+  # Doesn't play nice with ZFS in NixOS 25.05
+  zramSwap.enable = lib.mkForce false;
 }

morph/lib/base.nix

+# This module contains settings and configuration that apply to all nodes in a grid.
+{ lib, config, ...}:
+{
+  options.grid = {
+    publicKeyPath = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+      A path on the deployment system of a directory containing all of the
+      public keys for the system.  For example, this holds Wireguard public keys
+      for the VPN configuration and SSH public keys to configure SSH
+      authentication.
+      '';
+    };
+    privateKeyPath = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+      A path on the deployment system of a directory containing all of the
+      corresponding private keys for the system.
+      '';
+    };
+    monitoringvpnIPv4 = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The IPv4 address of this node on the monitoring VPN.
+      '';
+    };
+    monitoringvpnEndpoint = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The domain name and port of the monitoring VPN endpoint.
+      '';
+    };
+
+    letsEncryptAdminEmail = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        A string giving an email address to use for Let's Encrypt registration and
+        certificate issuance.
+      '';
+    };
+  };
+
+  # Any extra NixOS modules to load on all our servers.  Note that just
+  # because they're loaded doesn't *necessarily* mean they're turned on.
+  imports = [
+    # Set options intended for a "small" NixOS: Do not build X and docs.
+    <nixpkgs/nixos/modules/profiles/minimal.nix>
+    # This brings in various other modules that define options for different
+    # areas of the service.
+    ../../nixos/modules/default.nix
+  ];
+
+  config = {
+    # The morph default deployment target the name of the node in the network
+    # attrset.  We don't always want to give the node its proper public address
+    # there (because it depends on which domain is associated with the grid
+    # being configured and using variable names complicates a lot of things).
+    # Instead, just tell morph how to reach the node here - by using its fully
+    # qualified domain name.
+    deployment.targetHost = config.networking.fqdn;
+
+    services.private-storage.monitoring.exporters.promtail.enable = true;
+
+    # Install no documentation on grid
+    # It seems 24.05 has some new defaults that aren't stripped away by the
+    # 'minimal' profile above.
+    # See https://github.com/NixOS/nixpkgs/blob/nixos-24.05/nixos/modules/misc/documentation.nix
+    documentation.enable = false;
+    documentation.man.enable = false;
+    documentation.info.enable = false;
+    documentation.doc.enable = false;
+    documentation.dev.enable = false;
+    documentation.nixos.enable = false;
+
+    assertions = [
+      # This is a check to save somebody in the future trying to debug why
+      # setting `nixpkgs.config` is not having an effect.
+      {
+        # `{}` is the default value for `nixpkgs.config`
+        assertion = config.nixpkgs.config == {};
+        message = ''
+          Since we set `nixpkgs.pkgs` via morph's `network.pkgs`, the value for `nixpkgs.config` is ignored.
+          See https://whetstone.private.storage/privatestorage/PrivateStorageio/-/issues/85#note_15876 for details.
+          '';
+      }
+    ];
+  };
+}

morph/lib/bootstrap-configuration.nix

@@ -67,7 +67,7 @@ let
   # Stop!  I hope you're done when you get here.  If you have to modify
   # anything below this point the expression should probably be refactored and
   # another variable added controlling whatever new thing you need to control.
-  # Open an issue: https://github.com/PrivateStorageio/PrivateStorageio/issues/new
+  # Open an issue: https://whetstone.private.storage/privatestorage/PrivateStorageio/-/issues/new?issue
 in
 # Define a function that ignores all its arguments.  We don't need any of them
 # for now.
@@ -87,7 +87,6 @@ in
   # Configure the bootloader how we like.
   boot.loader.timeout = 10;
   boot.loader.grub.enable = true;
-  boot.loader.grub.version = 2;
   boot.loader.grub.device = "/dev/disk/by-id/${grubDeviceID}";
 
   # Let me in to do subsequent configuration.  This makes the machine wide

morph/lib/borgbackup.nix

+# Importing this adds a daily borg backup job to a node.
+# It has all the common config and keys, and can be configured
+# to back up more (or entirely different) folders.
+
+
+{ lib, config, pkgs, ...}:
+let
+  cfg = config.services.private-storage.borgbackup;
+  inherit (config.grid) publicKeyPath privateKeyPath;
+
+  # For each host generate a number between 0 and 15 so backup
+  # jobs don't all run at the same time.
+  ip-util = import ../../nixos/lib/ip-util.nix;
+  backupDelay = with builtins; bitAnd (ip-util.fromHexString
+    (substring 0 6 (hashString "md5" config.networking.hostName))) 15;
+
+in {
+  options.services.private-storage.borgbackup = {
+    enable = lib.mkEnableOption "Borgbackup daily backup job";
+    paths = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        A list of directories to back up using Borg.
+      '';
+      default = [ "/storage" ];
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    deployment = {
+      secrets = {
+        "borgbackup-passphrase" = {
+          # The passphrase is used to encrypt the repo key
+          # https://borgbackup.readthedocs.io/en/stable/usage/init.html
+          destination = "/run/keys/borgbackup/passphrase";
+          source = "${privateKeyPath}/borgbackup.passphrase";
+        };
+        "borgbackup-appendonly-ssh-key" = {
+          # The ssh key is used to authenticate to the remote repo server
+          destination = "/run/keys/borgbackup/ssh-key";
+          source = "${privateKeyPath}/borgbackup.ssh-key";
+        };
+      };
+    };
+
+    services.borgbackup.jobs = {
+      daily = {
+        paths = cfg.paths;
+        repo = lib.fileContents "${publicKeyPath}/borgbackup/${config.networking.hostName}.repopath";
+        doInit = false;
+        encryption = {
+          mode = "repokey-blake2";
+          passCommand = "cat /run/keys/borgbackup/passphrase";
+        };
+        environment = {
+          BORG_RSH = "ssh -i /run/keys/borgbackup/ssh-key -o StrictHostKeyChecking=accept-new";
+        };
+
+	# Output statistics after uploading a backup set
+	extraCreateArgs = "--stats --json";
+	# All logs in JSON to help Prometheus/Grafana
+	extraArgs = "--log-json";
+
+        # Ciphertext doesn't compress well
+        compression = "none";
+
+        # Start the backup at a different time per machine,
+        # and not at the full hour, but somewhat later
+        startAt = "*-*-* " + toString backupDelay + ":22:11 UTC";
+      };
+    };
+
+    # Check repo once a month
+    systemd.services.borgbackup-check-repo = {
+      # Once a month, 3h after last backup started.
+      # Add "1" because day 0 is invalid.
+      startAt = "*-*-" + toString (backupDelay + 1) + " 18:33:22 UTC";
+      path = [ pkgs.borgbackup ];
+      environment = {
+        BORG_PASSCOMMAND = "cat /run/keys/borgbackup/passphrase";
+        BORG_RSH = "ssh -i /run/keys/borgbackup/ssh-key -o StrictHostKeyChecking=accept-new";
+        BORG_REPO = lib.fileContents "${publicKeyPath}/borgbackup/${config.networking.hostName}.repopath";
+      };
+      script = ''${pkgs.borgbackup}/bin/borg check --verbose --log-json'';
+    };
+  };
+}

morph/lib/customize-issuer.nix

-# Define a function which returns a value which fills in all the holes left by
-# ``issuer.nix``.
-{
-  # A path on the deployment system of a directory containing all of the
-  # public keys for the system.  For example, this holds Wireguard public keys
-  # for the VPN configuration and SSH public keys to configure SSH
-  # authentication.
-  publicKeyPath
-
-  # A path on the deployment system of a directory containing all of the
-  # corresponding private keys for the system.
-, privateKeyPath
-
-  # A string giving the IP address and port number (":"-separated) of the VPN
-  # server.
-, monitoringvpnEndpoint
-
-  # A string giving the VPN IPv4 address for this system.
-, monitoringvpnIPv4
-
-  # A string giving the domain name associated with this grid.  This is meant
-  # to be combined with the hostname for this system to produce a
-  # fully-qualified domain name.  For example, an issuer might have "payments"
-  # as its hostname and belong to a grid with the domain
-  # "example-grid.invalid".  This ``domain`` parameter should have the value
-  # ``"example-grid.invalid"`` for the system figure out that
-  # ``payments.example-grid.invalid`` is the name of this system.
-, domain
-
-  # A set mapping usernames as strings to SSH public keys as strings.  For
-  # each element of the site, the indicated user is configured on the system
-  # with the indicated SSH key as an authorized key.
-, sshUsers
-
-  # A string giving an email address to use for Let's Encrypt registration and
-  # certificate issuance.
-, letsEncryptAdminEmail
-
-  # A list of strings giving the domain names that point at this issuer
-  # system.  These will all be included in Let's Encrypt certificate.
-, issuerDomains
-
-  # A list of strings giving CORS Origins will the issuer will be configured
-  # to allow.
-, allowedChargeOrigins
-, ...
-}:
-{ config, ... }: {
-  # The morph default deployment target the name of the node in the network
-  # attrset.  We don't always want to give the node its proper public address
-  # there (because it depends on which domain is associated with the grid
-  # being configured and using variable names complicates a lot of things).
-  # Instead, just tell morph how to reach the node here - by using its fully
-  # qualified domain name.
-  deployment.targetHost = "${config.networking.hostName}.${config.networking.domain}";
-
-  deployment.secrets = {
-    # A path on the deployment system to a file containing the Ristretto
-    # signing key.  This is used as the source of the Ristretto signing key
-    # morph secret.
-    "ristretto-signing-key".source = "${privateKeyPath}/ristretto.signing-key";
-
-    # A path on the deployment system to a file containing the Stripe secret
-    # key.  This is used as the source of the Stripe secret key morph secret.
-    "stripe-secret-key".source = "${privateKeyPath}/stripe.secret";
-
-    # ``.../monitoringvpn`` is a path on the deployment system of a directory
-    # containing a number of VPN-related secrets.  This is expected to contain
-    # a number of files named like ``<VPN IPv4 address>.key`` containing the
-    # VPN private key for the corresponding host.  It must also contain
-    # ``server.pub`` and ``preshared.key`` holding the VPN server's public key
-    # and the pre-shared key, respectively.  All of these things are used as
-    # the sources of various VPN-related morph secrets.
-    "monitoringvpn-secret-key".source = "${privateKeyPath}/monitoringvpn/${monitoringvpnIPv4}.key";
-    "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
-  };
-
-  networking.domain = domain;
-
-  services.private-storage.sshUsers = sshUsers;
-  services.private-storage.monitoring.vpn.client = {
-    enable = true;
-    ip = monitoringvpnIPv4;
-    endpoint = monitoringvpnEndpoint;
-    endpointPublicKeyFile = "${publicKeyPath}/monitoringvpn/server.pub";
-  };
-
-  services.private-storage-issuer = {
-    inherit letsEncryptAdminEmail allowedChargeOrigins;
-    domains = issuerDomains;
-  };
-
-  system.stateVersion = "19.03";
-}

morph/lib/customize-monitoring.nix

-# Define a function which returns a value which fills in all the holes left by
-# ``monitoring.nix``.
-{
-  # A set mapping VPN IP addresses as strings to lists of hostnames as
-  # strings.  The system's ``/etc/hosts`` will be populated with this
-  # information.  Apart from helping with normal forward resolution, this
-  # *also* gives us reverse resolution from the VPN IPs to hostnames which
-  # allows Grafana to show us hostnames instead of VPN IP addresses.
-  hostsMap
-
-  # See ``customize-issuer.nix``.
-, publicKeyPath
-, privateKeyPath
-, monitoringvpnIPv4
-, domain
-
-  # A list of VPN IP addresses as strings indicating which clients will be
-  # allowed onto the VPN.
-, vpnClientIPs
-
-  # A list of VPN clients (IP addresses or hostnames) as strings indicating
-  # which nodes to scrape "nodeExporter" metrics from.
-, nodeExporterTargets
-
-  # A list of VPN clients (IP addresses or hostnames) as strings indicating
-  # which nodes to scrape "nginxExporter" metrics from.
-, nginxExporterTargets ? []
-
-  # A list of VPN clients (IP addresses or hostnames) as strings indicating
-  # which nodes to scrape PaymentServer metrics from.
-, paymentExporterTargets ? []
-
-  # A string giving the NixOS state version for the system.
-, stateVersion
-, ...
-}:
-{ config, ... }: {
-  # See customize-issuer.nix for an explanatoin of targetHost value.
-  deployment.targetHost = "${config.networking.hostName}.${config.networking.domain}";
-
-  deployment.secrets = {
-    "monitoringvpn-private-key".source = "${privateKeyPath}/monitoringvpn/server.key";
-    "monitoringvpn-preshared-key".source = "${privateKeyPath}/monitoringvpn/preshared.key";
-  };
-
-  networking.domain = domain;
-  networking.hosts = hostsMap;
-
-  services.private-storage.monitoring.vpn.server = {
-    enable = true;
-    ip = monitoringvpnIPv4;
-    inherit vpnClientIPs;
-    pubKeysPath = "${publicKeyPath}/monitoringvpn";
-  };
-
-  services.private-storage.monitoring.prometheus = {
-    inherit nodeExporterTargets;
-    inherit nginxExporterTargets;
-    inherit paymentExporterTargets;
-  };
-
-  system.stateVersion = stateVersion;
-}